Subzero. Enable Atomics in ARM.

Makefile.standalone

@@ -383,7 +383,6 @@ check-xtest: $(OBJDIR)/pnacl-sz make_symlink runtime
 	  -e x8664,native,sse4.1,test_vector_ops \
 	  -e x8664,native,sse2,test_global \
 	  -i arm32,native,neon \
-	  -e arm32,native,neon,test_sync_atomic \
 	  -e arm32,native,neon,test_vector_ops \
 	  -e arm32,native,neon,test_select
 	PNACL_BIN_PATH=$(PNACL_BIN_PATH) \

crosstest/test_sync_atomic_main.cpp

@@ -92,7 +92,8 @@ void testAtomicRMW(volatile Type *AtomicLoc, size_t &TotalTests, size_t &Passes,
           } else {
             ++Failures;
             std::cout << "test_" << Funcs[f].Name << (CHAR_BIT * sizeof(Type))
-                      << "(" << static_cast<uint64>(Value1) << ", "
+                      << "(" << fetch_first << ", "
+                      << static_cast<uint64>(Value1) << ", "
                       << static_cast<uint64>(Value2)
                       << "): sz1=" << static_cast<uint64>(ResultSz1)
                       << " llc1=" << static_cast<uint64>(ResultLlc1)
@@ -139,7 +140,7 @@ void testValCompareAndSwap(volatile Type *AtomicLoc, size_t &TotalTests,
             ++Failures;
             std::cout << "test_" << Funcs[f].Name << (CHAR_BIT * sizeof(Type))
                       << "(" << static_cast<uint64>(Value1) << ", "
-                      << static_cast<uint64>(Value2)
+                      << static_cast<uint64>(Value2) << ", flip=" << flip
                       << "): sz1=" << static_cast<uint64>(ResultSz1)
                       << " llc1=" << static_cast<uint64>(ResultLlc1)
                       << " sz2=" << static_cast<uint64>(ResultSz2)
@@ -159,7 +160,15 @@ template <typename Type> struct ThreadData {
 };
 
 template <typename Type> void *threadWrapper(void *Data) {
-  const size_t NumReps = 8000;
+#ifdef ARM32
+  // Given that most of times these crosstests for ARM are run under qemu, we
+  // set a lower NumReps to allow crosstests to complete within a reasonable
+  // amount of time.
+  static const size_t NumReps = 1000;
+#else  // ARM32
+  static const size_t NumReps = 8000;
+#endif // ARM32
+
   ThreadData<Type> *TData = reinterpret_cast<ThreadData<Type> *>(Data);
   for (size_t i = 0; i < NumReps; ++i) {
     (void)TData->FuncPtr(TData->Fetch, TData->Ptr, TData->Adjustment);

src/IceTargetLoweringARM32.cpp

@@ -195,7 +195,6 @@ TargetARM32::TargetARM32(Cfg *Func)
            "Duplicate alias for " #val);                                       \
     RegisterAliases[RegARM32::val].set(RegAlias);                              \
   }                                                                            \
-  RegisterAliases[RegARM32::val].resize(RegARM32::Reg_NUM);                    \
   assert(RegisterAliases[RegARM32::val][RegARM32::val]);                       \
   ScratchRegs[RegARM32::val] = scratch;
   REGARM32_TABLE;
@@ -217,6 +216,34 @@ TargetARM32::TargetARM32(Cfg *Func)
   TypeToRegisterSet[IceType_v4f32] = VectorRegisters;
 }
 
+namespace {
+void copyRegAllocFromInfWeightVariable64On32(const VarList &Vars) {
+  for (Variable *Var : Vars) {
+    auto *Var64 = llvm::dyn_cast<Variable64On32>(Var);
+    if (!Var64) {
+      // This is not the variable we are looking for.
+      continue;
+    }
+    assert(Var64->hasReg() || !Var64->mustHaveReg());
+    if (!Var64->hasReg()) {
+      continue;
+    }
+    SizeT FirstReg = RegARM32::getI64PairFirstGPRNum(Var->getRegNum());
+    // This assumes little endian.
+    Variable *Lo = Var64->getLo();
+    Variable *Hi = Var64->getHi();
+    assert(Lo->hasReg() == Hi->hasReg());
+    if (Lo->hasReg()) {
+      continue;
+    }
+    Lo->setRegNum(FirstReg);
+    Lo->setMustHaveReg();
+    Hi->setRegNum(FirstReg + 1);
+    Hi->setMustHaveReg();
+  }
+}
+} // end of anonymous namespace
+
 void TargetARM32::translateO2() {
   TimerMarker T(TimerStack::TT_O2, Func);
 
@@ -284,6 +311,7 @@ void TargetARM32::translateO2() {
   regAlloc(RAK_Global);
   if (Func->hasError())
     return;
+  copyRegAllocFromInfWeightVariable64On32(Func->getVariables());
   Func->dump("After linear scan regalloc");
 
   if (Ctx->getFlags().getPhiEdgeSplit()) {
@@ -344,6 +372,7 @@ void TargetARM32::translateOm1() {
   regAlloc(RAK_InfOnly);
   if (Func->hasError())
     return;
+  copyRegAllocFromInfWeightVariable64On32(Func->getVariables());
   Func->dump("After regalloc of infinite-weight variables");
 
   Func->genFrame();
@@ -616,7 +645,7 @@ void TargetARM32::finishArgumentLowering(Variable *Arg, Variable *FramePtr,
     auto *Mem = OperandARM32Mem::create(
         Func, Ty, FramePtr, llvm::cast<ConstantInteger32>(
                                 Ctx->getConstantInt32(Arg->getStackOffset())));
-    legalizeToReg(Mem, Arg->getRegNum());
+    _mov(Arg, legalizeToReg(Mem, Arg->getRegNum()));
     // This argument-copying instruction uses an explicit OperandARM32Mem
     // operand instead of a Variable, so its fill-from-stack operation has to
     // be tracked separately for statistics.
@@ -716,6 +745,11 @@ void TargetARM32::addProlog(CfgNode *Node) {
     RegsUsed[RegARM32::Reg_lr] = true;
   }
   for (SizeT i = 0; i < CalleeSaves.size(); ++i) {
+    if (RegARM32::isI64RegisterPair(i)) {
+      // We don't save register pairs explicitly. Instead, we rely on the code
+      // fake-defing/fake-using each register in the pair.
+      continue;
+    }
     if (CalleeSaves[i] && RegsUsed[i]) {
       // TODO(jvoung): do separate vpush for each floating point register
       // segment and += 4, or 8 depending on type.
@@ -884,6 +918,10 @@ void TargetARM32::addEpilog(CfgNode *Node) {
   // Pop registers in ascending order just like push (instead of in reverse
   // order).
   for (SizeT i = 0; i < CalleeSaves.size(); ++i) {
+    if (RegARM32::isI64RegisterPair(i)) {
+      continue;
+    }
+
     if (CalleeSaves[i] && RegsUsed[i]) {
       GPRsToRestore.push_back(getPhysicalRegister(i));
     }
@@ -1739,6 +1777,7 @@ void TargetARM32::lowerAssign(const InstAssign *Inst) {
     Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
     Variable *T_Lo = makeReg(IceType_i32);
     Variable *T_Hi = makeReg(IceType_i32);
+
     _mov(T_Lo, Src0Lo);
     _mov(DestLo, T_Lo);
     _mov(T_Hi, Src0Hi);
@@ -2271,9 +2310,7 @@ void TargetARM32::lowerCast(const InstCast *Inst) {
       configureBitcastTemporary(T);
       Variable *Src0R = legalizeToReg(Src0);
       _mov(T, Src0R);
-      auto *Dest64On32 = llvm::cast<Variable64On32>(Dest);
-      lowerAssign(InstAssign::create(Func, Dest64On32->getLo(), T->getLo()));
-      lowerAssign(InstAssign::create(Func, Dest64On32->getHi(), T->getHi()));
+      lowerAssign(InstAssign::create(Func, Dest, T));
       break;
     }
     case IceType_f64: {
@@ -2282,11 +2319,11 @@ void TargetARM32::lowerCast(const InstCast *Inst) {
       // vmov T2, T0, T1
       // Dest <- T2
       assert(Src0->getType() == IceType_i64);
+      Variable *T = makeReg(DestType);
       auto *Src64 = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
       Src64->initHiLo(Func);
       configureBitcastTemporary(Src64);
       lowerAssign(InstAssign::create(Func, Src64, Src0));
-      Variable *T = makeReg(IceType_f64);
       _mov(T, Src64);
       lowerAssign(InstAssign::create(Func, Dest, T));
       break;
@@ -2537,38 +2574,460 @@ void TargetARM32::lowerInsertElement(const InstInsertElement *Inst) {
   UnimplementedError(Func->getContext()->getFlags());
 }
 
-void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
-  switch (Instr->getIntrinsicInfo().ID) {
-  case Intrinsics::AtomicCmpxchg: {
-    UnimplementedError(Func->getContext()->getFlags());
+namespace {
+inline uint64_t getConstantMemoryOrder(Operand *Opnd) {
+  if (auto Integer = llvm::dyn_cast<ConstantInteger32>(Opnd))
+    return Integer->getValue();
+  return Intrinsics::MemoryOrderInvalid;
+}
+} // end of anonymous namespace
+
+void TargetARM32::lowerAtomicRMW(Variable *Dest, uint32_t Operation,
+                                 Operand *Ptr, Operand *Val) {
+  // retry:
+  //     ldrex contents, [addr]
+  //     op tmp, contents, operand
+  //     strex success, tmp, [addr]
+  //     jne retry
+  //     fake-use(addr, operand)  @ prevents undesirable clobbering.
+  //     mov dest, contents
+  assert(Dest != nullptr);
+  Type DestTy = Dest->getType();
+  (void)Ptr;
+  (void)Val;
+
+  OperandARM32Mem *Mem;
+  Variable *PtrContentsReg;
+  Variable *PtrContentsHiReg;
+  Variable *PtrContentsLoReg;
+  Variable *Value = Func->makeVariable(DestTy);
+  Variable *ValueReg;
+  Variable *ValueHiReg;
+  Variable *ValueLoReg;
+  Variable *Success = makeReg(IceType_i32);
+  Variable *TmpReg;
+  Variable *TmpHiReg;
+  Variable *TmpLoReg;
+  Operand *_0 = Ctx->getConstantZero(IceType_i32);
+  InstARM32Label *Retry = InstARM32Label::create(Func, this);
+
+  if (DestTy == IceType_i64) {
+    Variable64On32 *PtrContentsReg64 = makeI64RegPair();
+    PtrContentsHiReg = PtrContentsReg64->getHi();
+    PtrContentsLoReg = PtrContentsReg64->getLo();
+    PtrContentsReg = PtrContentsReg64;
+
+    llvm::cast<Variable64On32>(Value)->initHiLo(Func);
+    Variable64On32 *ValueReg64 = makeI64RegPair();
+    ValueHiReg = ValueReg64->getHi();
+    ValueLoReg = ValueReg64->getLo();
+    ValueReg = ValueReg64;
+
+    Variable64On32 *TmpReg64 = makeI64RegPair();
+    TmpHiReg = TmpReg64->getHi();
+    TmpLoReg = TmpReg64->getLo();
+    TmpReg = TmpReg64;
+  } else {
+    PtrContentsReg = makeReg(DestTy);
+    PtrContentsHiReg = nullptr;
+    PtrContentsLoReg = PtrContentsReg;
+
+    ValueReg = makeReg(DestTy);
+    ValueHiReg = nullptr;
+    ValueLoReg = ValueReg;
+
+    TmpReg = makeReg(DestTy);
+    TmpHiReg = nullptr;
+    TmpLoReg = TmpReg;
+  }
+
+  if (DestTy == IceType_i64) {
+    Context.insert(InstFakeDef::create(Func, Value));
+  }
+  lowerAssign(InstAssign::create(Func, Value, Val));
+
+  Variable *PtrVar = Func->makeVariable(IceType_i32);
+  lowerAssign(InstAssign::create(Func, PtrVar, Ptr));
+
+  _dmb();
+  Context.insert(Retry);
+  Mem = formMemoryOperand(PtrVar, DestTy);
+  if (DestTy == IceType_i64) {
+    Context.insert(InstFakeDef::create(Func, ValueReg, Value));
+  }
+  lowerAssign(InstAssign::create(Func, ValueReg, Value));
+  if (DestTy == IceType_i8 || DestTy == IceType_i16) {
+    _uxt(ValueReg, ValueReg);
+  }
+  _ldrex(PtrContentsReg, Mem);
+
+  if (DestTy == IceType_i64) {
+    Context.insert(InstFakeDef::create(Func, TmpReg, ValueReg));
+  }
+  switch (Operation) {
+  default:
+    Func->setError("Unknown AtomicRMW operation");
     return;
+  case Intrinsics::AtomicAdd:
+    if (DestTy == IceType_i64) {
+      _adds(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+      _adc(TmpHiReg, PtrContentsHiReg, ValueHiReg);
+    } else {
+      _add(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+    }
+    break;
+  case Intrinsics::AtomicSub:
+    if (DestTy == IceType_i64) {
+      _subs(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+      _sbc(TmpHiReg, PtrContentsHiReg, ValueHiReg);
+    } else {
+      _sub(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+    }
+    break;
+  case Intrinsics::AtomicOr:
+    _orr(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+    if (DestTy == IceType_i64) {
+      _orr(TmpHiReg, PtrContentsHiReg, ValueHiReg);
+    }
+    break;
+  case Intrinsics::AtomicAnd:
+    _and(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+    if (DestTy == IceType_i64) {
+      _and(TmpHiReg, PtrContentsHiReg, ValueHiReg);
+    }
+    break;
+  case Intrinsics::AtomicXor:
+    _eor(TmpLoReg, PtrContentsLoReg, ValueLoReg);
+    if (DestTy == IceType_i64) {
+      _eor(TmpHiReg, PtrContentsHiReg, ValueHiReg);
+    }
+    break;
+  case Intrinsics::AtomicExchange:
+    _mov(TmpLoReg, ValueLoReg);
+    if (DestTy == IceType_i64) {
+      _mov(TmpHiReg, ValueHiReg);
+    }
+    break;
   }
+  _strex(Success, TmpReg, Mem);
+  _cmp(Success, _0);
+  _br(Retry, CondARM32::NE);
+
+  // The following fake-uses ensure that Subzero will not clobber them in the
+  // load-linked/store-conditional loop above. We might have to spill them, but
+  // spilling is preferable over incorrect behavior.
+  Context.insert(InstFakeUse::create(Func, PtrVar));
+  if (auto *Value64 = llvm::dyn_cast<Variable64On32>(Value)) {
+    Context.insert(InstFakeUse::create(Func, Value64->getHi()));
+    Context.insert(InstFakeUse::create(Func, Value64->getLo()));
+  } else {
+    Context.insert(InstFakeUse::create(Func, Value));
+  }
+  _dmb();
+  if (DestTy == IceType_i8 || DestTy == IceType_i16) {
+    _uxt(PtrContentsReg, PtrContentsReg);
+  }
+
+  if (DestTy == IceType_i64) {
+    Context.insert(InstFakeUse::create(Func, PtrContentsReg));
+  }
+  lowerAssign(InstAssign::create(Func, Dest, PtrContentsReg));
+  if (auto *Dest64 = llvm::dyn_cast<Variable64On32>(Dest)) {
+    Context.insert(InstFakeUse::create(Func, Dest64->getLo()));
+    Context.insert(InstFakeUse::create(Func, Dest64->getHi()));
+  } else {
+    Context.insert(InstFakeUse::create(Func, Dest));
+  }
+}
+
+void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
+  Variable *Dest = Instr->getDest();
+  Type DestTy = (Dest != nullptr) ? Dest->getType() : IceType_void;
+  Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID;
+  switch (ID) {
   case Intrinsics::AtomicFence:
-    UnimplementedError(Func->getContext()->getFlags());
-    return;
   case Intrinsics::AtomicFenceAll:
-    // NOTE: FenceAll should prevent and load/store from being moved across the
-    // fence (both atomic and non-atomic). The InstARM32Mfence instruction is
-    // currently marked coarsely as "HasSideEffects".
-    UnimplementedError(Func->getContext()->getFlags());
+    assert(Dest == nullptr);
+    _dmb();
     return;
   case Intrinsics::AtomicIsLockFree: {
-    UnimplementedError(Func->getContext()->getFlags());
+    Operand *ByteSize = Instr->getArg(0);
+    auto *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize);
+    if (CI == nullptr) {
+      // The PNaCl ABI requires the byte size to be a compile-time constant.
+      Func->setError("AtomicIsLockFree byte size should be compile-time const");
+      return;
+    }
+    static constexpr int32_t NotLockFree = 0;
+    static constexpr int32_t LockFree = 1;
+    int32_t Result = NotLockFree;
+    switch (CI->getValue()) {
+    case 1:
+    case 2:
+    case 4:
+    case 8:
+      Result = LockFree;
+      break;
+    }
+    _mov(Dest, legalizeToReg(Ctx->getConstantInt32(Result)));
     return;
   }
   case Intrinsics::AtomicLoad: {
-    UnimplementedError(Func->getContext()->getFlags());
+    assert(isScalarIntegerType(DestTy));
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(1)))) {
+      Func->setError("Unexpected memory ordering for AtomicLoad");
+      return;
+    }
+    Variable *T;
+
+    if (DestTy == IceType_i64) {
+      // ldrex is the only arm instruction that is guaranteed to load a 64-bit
+      // integer atomically. Everything else works with a regular ldr.
+      T = makeI64RegPair();
+      _ldrex(T, formMemoryOperand(Instr->getArg(0), IceType_i64));
+    } else {
+      T = makeReg(DestTy);
+      _ldr(T, formMemoryOperand(Instr->getArg(0), DestTy));
+    }
+    _dmb();
+    lowerAssign(InstAssign::create(Func, Dest, T));
+    // Make sure the atomic load isn't elided when unused, by adding a FakeUse.
+    // Since lowerLoad may fuse the load w/ an arithmetic instruction, insert
+    // the FakeUse on the last-inserted instruction's dest.
+    Context.insert(
+        InstFakeUse::create(Func, Context.getLastInserted()->getDest()));
     return;
   }
-  case Intrinsics::AtomicRMW:
-    UnimplementedError(Func->getContext()->getFlags());
-    return;
   case Intrinsics::AtomicStore: {
-    UnimplementedError(Func->getContext()->getFlags());
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(2)))) {
+      Func->setError("Unexpected memory ordering for AtomicStore");
+      return;
+    }
+    Operand *Value = Instr->getArg(0);
+    Type ValueTy = Value->getType();
+    assert(isScalarIntegerType(ValueTy));
+    Operand *Addr = Instr->getArg(1);
+
+    if (ValueTy == IceType_i64) {
+      // Atomic 64-bit stores require a load-locked/store-conditional loop using
+      // ldrexd, and strexd. The lowered code is:
+      //
+      // retry:
+      //     ldrexd t.lo, t.hi, [addr]
+      //     strexd success, value.lo, value.hi, [addr]
+      //     cmp success, #0
+      //     bne retry
+      //     fake-use(addr, value.lo, value.hi)
+      //
+      // The fake-use is needed to prevent those variables from being clobbered
+      // in the loop (which will happen under register pressure.)
+      Variable64On32 *Tmp = makeI64RegPair();
+      Variable64On32 *ValueVar =
+          llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+      Variable *AddrVar = makeReg(IceType_i32);
+      Variable *Success = makeReg(IceType_i32);
+      OperandARM32Mem *Mem;
+      Operand *_0 = Ctx->getConstantZero(IceType_i32);
+      InstARM32Label *Retry = InstARM32Label::create(Func, this);
+      Variable64On32 *NewReg = makeI64RegPair();
+      ValueVar->initHiLo(Func);
+      ValueVar->mustNotHaveReg();
+
+      _dmb();
+      lowerAssign(InstAssign::create(Func, ValueVar, Value));
+      lowerAssign(InstAssign::create(Func, AddrVar, Addr));
+
+      Context.insert(Retry);
+      Context.insert(InstFakeDef::create(Func, NewReg));
+      lowerAssign(InstAssign::create(Func, NewReg, ValueVar));
+      Mem = formMemoryOperand(AddrVar, IceType_i64);
+      _ldrex(Tmp, Mem);
+      // This fake-use both prevents the ldrex from being dead-code eliminated,
+      // while also keeping liveness happy about all defs being used.
+      Context.insert(
+          InstFakeUse::create(Func, Context.getLastInserted()->getDest()));
+      _strex(Success, NewReg, Mem);
+      _cmp(Success, _0);
+      _br(Retry, CondARM32::NE);
+
+      Context.insert(InstFakeUse::create(Func, ValueVar->getLo()));
+      Context.insert(InstFakeUse::create(Func, ValueVar->getHi()));
+      Context.insert(InstFakeUse::create(Func, AddrVar));
+      _dmb();
+      return;
+    }
+    // non-64-bit stores are atomically as long as the address is aligned. This
+    // is PNaCl, so addresses are aligned.
+    Variable *T = makeReg(ValueTy);
+
+    _dmb();
+    lowerAssign(InstAssign::create(Func, T, Value));
+    _str(T, formMemoryOperand(Addr, ValueTy));
+    _dmb();
+    return;
+  }
+  case Intrinsics::AtomicCmpxchg: {
+    // The initial lowering for cmpxchg was:
+    //
+    // retry:
+    //     ldrex tmp, [addr]
+    //     cmp tmp, expected
+    //     mov expected, tmp
+    //     jne retry
+    //     strex success, new, [addr]
+    //     cmp success, #0
+    //     bne retry
+    //     mov dest, expected
+    //
+    // Besides requiring two branches, that lowering could also potentially
+    // write to memory (in mov expected, tmp) unless we were OK with increasing
+    // the register pressure and requiring expected to be an infinite-weight
+    // variable (spoiler alert: that was a problem for i64 cmpxchg.) Through
+    // careful rewritting, and thanks to predication, we now implement the
+    // lowering as:
+    //
+    // retry:
+    //     ldrex tmp, [addr]
+    //     cmp tmp, expected
+    //     strexeq success, new, [addr]
+    //     movne expected, tmp
+    //     cmpeq success, #0
+    //     bne retry
+    //     mov dest, expected
+    //
+    // Predication lets us move the strex ahead of the mov expected, tmp, which
+    // allows tmp to be a non-infinite weight temporary. We wanted to avoid
+    // writing to memory between ldrex and strex because, even though most times
+    // that would cause no issues, if any interleaving memory write aliased
+    // [addr] than we would have undefined behavior. Undefined behavior isn't
+    // cool, so we try to avoid it. See the "Synchronization and semaphores"
+    // section of the "ARM Architecture Reference Manual."
+
+    assert(isScalarIntegerType(DestTy));
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)),
+            getConstantMemoryOrder(Instr->getArg(4)))) {
+      Func->setError("Unexpected memory ordering for AtomicCmpxchg");
+      return;
+    }
+
+    OperandARM32Mem *Mem;
+    Variable *TmpReg;
+    Variable *Expected, *ExpectedReg;
+    Variable *New, *NewReg;
+    Variable *Success = makeReg(IceType_i32);
+    Operand *_0 = Ctx->getConstantZero(IceType_i32);
+    InstARM32Label *Retry = InstARM32Label::create(Func, this);
+
+    if (DestTy == IceType_i64) {
+      Variable64On32 *TmpReg64 = makeI64RegPair();
+      Variable64On32 *New64 =
+          llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+      Variable64On32 *NewReg64 = makeI64RegPair();
+      Variable64On32 *Expected64 =
+          llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+      Variable64On32 *ExpectedReg64 = makeI64RegPair();
+
+      New64->initHiLo(Func);
+      New64->mustNotHaveReg();
+      Expected64->initHiLo(Func);
+      Expected64->mustNotHaveReg();
+
+      TmpReg = TmpReg64;
+      New = New64;
+      NewReg = NewReg64;
+      Expected = Expected64;
+      ExpectedReg = ExpectedReg64;
+    } else {
+      TmpReg = makeReg(DestTy);
+      New = Func->makeVariable(DestTy);
+      NewReg = makeReg(DestTy);
+      Expected = Func->makeVariable(DestTy);
+      ExpectedReg = makeReg(DestTy);
+    }
+
+    Mem = formMemoryOperand(Instr->getArg(0), DestTy);
+    if (DestTy == IceType_i64) {
+      Context.insert(InstFakeDef::create(Func, Expected));
+    }
+    lowerAssign(InstAssign::create(Func, Expected, Instr->getArg(1)));
+    if (DestTy == IceType_i64) {
+      Context.insert(InstFakeDef::create(Func, New));
+    }
+    lowerAssign(InstAssign::create(Func, New, Instr->getArg(2)));
+    _dmb();
+
+    Context.insert(Retry);
+    if (DestTy == IceType_i64) {
+      Context.insert(InstFakeDef::create(Func, ExpectedReg, Expected));
+    }
+    lowerAssign(InstAssign::create(Func, ExpectedReg, Expected));
+    if (DestTy == IceType_i64) {
+      Context.insert(InstFakeDef::create(Func, NewReg, New));
+    }
+    lowerAssign(InstAssign::create(Func, NewReg, New));
+
+    _ldrex(TmpReg, Mem);
+    Context.insert(
+        InstFakeUse::create(Func, Context.getLastInserted()->getDest()));
+    if (DestTy == IceType_i64) {
+      auto *TmpReg64 = llvm::cast<Variable64On32>(TmpReg);
+      auto *ExpectedReg64 = llvm::cast<Variable64On32>(ExpectedReg);
+      // lowerAssign above has added fake-defs for TmpReg and ExpectedReg. Let's
+      // keep liveness happy, shall we?
+      Context.insert(InstFakeUse::create(Func, TmpReg));
+      Context.insert(InstFakeUse::create(Func, ExpectedReg));
+      _cmp(TmpReg64->getHi(), ExpectedReg64->getHi());
+      _cmp(TmpReg64->getLo(), ExpectedReg64->getLo(), CondARM32::EQ);
+    } else {
+      _cmp(TmpReg, ExpectedReg);
+    }
+    _strex(Success, NewReg, Mem, CondARM32::EQ);
+    if (DestTy == IceType_i64) {
+      auto *TmpReg64 = llvm::cast<Variable64On32>(TmpReg);
+      auto *Expected64 = llvm::cast<Variable64On32>(Expected);
+      _mov_redefined(Expected64->getHi(), TmpReg64->getHi(), CondARM32::NE);
+      _mov_redefined(Expected64->getLo(), TmpReg64->getLo(), CondARM32::NE);
+      auto *FakeDef = InstFakeDef::create(Func, Expected, TmpReg);
+      Context.insert(FakeDef);
+      FakeDef->setDestRedefined();
+    } else {
+      _mov_redefined(Expected, TmpReg, CondARM32::NE);
+    }
+    _cmp(Success, _0, CondARM32::EQ);
+    _br(Retry, CondARM32::NE);
+    _dmb();
+    lowerAssign(InstAssign::create(Func, Dest, Expected));
+    Context.insert(InstFakeUse::create(Func, Expected));
+    if (auto *New64 = llvm::dyn_cast<Variable64On32>(New)) {
+      Context.insert(InstFakeUse::create(Func, New64->getLo()));
+      Context.insert(InstFakeUse::create(Func, New64->getHi()));
+    } else {
+      Context.insert(InstFakeUse::create(Func, New));
+    }
+    return;
+  }
+  case Intrinsics::AtomicRMW: {
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)))) {
+      Func->setError("Unexpected memory ordering for AtomicRMW");
+      return;
+    }
+    lowerAtomicRMW(
+        Dest, static_cast<uint32_t>(
+                  llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue()),
+        Instr->getArg(1), Instr->getArg(2));
     return;
   }
   case Intrinsics::Bswap: {
-    Variable *Dest = Instr->getDest();
     Operand *Val = Instr->getArg(0);
     Type Ty = Val->getType();
     if (Ty == IceType_i64) {
@@ -2598,7 +3057,6 @@ void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
     return;
   }
   case Intrinsics::Ctpop: {
-    Variable *Dest = Instr->getDest();
     Operand *Val = Instr->getArg(0);
     InstCall *Call = makeHelperCall(isInt32Asserting32Or64(Val->getType())
                                         ? H_call_ctpop_i32
@@ -2633,7 +3091,7 @@ void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
     } else {
       ValLoR = legalizeToReg(Val);
     }
-    lowerCLZ(Instr->getDest(), ValLoR, ValHiR);
+    lowerCLZ(Dest, ValLoR, ValHiR);
     return;
   }
   case Intrinsics::Cttz: {
@@ -2657,17 +3115,16 @@ void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
       _rbit(T, ValLoR);
       ValLoR = T;
     }
-    lowerCLZ(Instr->getDest(), ValLoR, ValHiR);
+    lowerCLZ(Dest, ValLoR, ValHiR);
     return;
   }
   case Intrinsics::Fabs: {
-    Variable *Dest = Instr->getDest();
     Type DestTy = Dest->getType();
     Variable *T = makeReg(DestTy);
     if (isVectorType(DestTy)) {
       // Add a fake def to keep liveness consistent in the meantime.
       Context.insert(InstFakeDef::create(Func, T));
-      _mov(Instr->getDest(), T);
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       return;
     }
@@ -2721,20 +3178,19 @@ void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
     if (Ctx->getFlags().getUseSandboxing()) {
       UnimplementedError(Func->getContext()->getFlags());
     } else {
-      InstCall *Call = makeHelperCall(H_call_read_tp, Instr->getDest(), 0);
+      InstCall *Call = makeHelperCall(H_call_read_tp, Dest, 0);
       lowerCall(Call);
     }
     return;
   }
   case Intrinsics::Setjmp: {
-    InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1);
+    InstCall *Call = makeHelperCall(H_call_setjmp, Dest, 1);
     Call->addArg(Instr->getArg(0));
     lowerCall(Call);
     return;
   }
   case Intrinsics::Sqrt: {
     Variable *Src = legalizeToReg(Instr->getArg(0));
-    Variable *Dest = Instr->getDest();
     Variable *T = makeReg(Dest->getType());
     _vsqrt(T, Src);
     _mov(Dest, T);
@@ -2742,7 +3198,6 @@ void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
   }
   case Intrinsics::Stacksave: {
     Variable *SP = getPhysicalRegister(RegARM32::Reg_sp);
-    Variable *Dest = Instr->getDest();
     _mov(Dest, SP);
     return;
   }
@@ -3224,6 +3679,16 @@ OperandARM32Mem *TargetARM32::formMemoryOperand(Operand *Operand, Type Ty) {
       llvm::cast<ConstantInteger32>(Ctx->getConstantZero(IceType_i32)));
 }
 
+Variable64On32 *TargetARM32::makeI64RegPair() {
+  Variable64On32 *Reg =
+      llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+  Reg->setMustHaveReg();
+  Reg->initHiLo(Func);
+  Reg->getLo()->setMustNotHaveReg();
+  Reg->getHi()->setMustNotHaveReg();
+  return Reg;
+}
+
 Variable *TargetARM32::makeReg(Type Type, int32_t RegNum) {
   // There aren't any 64-bit integer registers for ARM32.
   assert(Type != IceType_i64);

src/IceTargetLoweringARM32.h

@@ -131,6 +131,8 @@ protected:
   void lowerExtractElement(const InstExtractElement *Inst) override;
   void lowerFcmp(const InstFcmp *Inst) override;
   void lowerIcmp(const InstIcmp *Inst) override;
+  void lowerAtomicRMW(Variable *Dest, uint32_t Operation, Operand *Ptr,
+                      Operand *Val);
   void lowerIntrinsicCall(const InstIntrinsicCall *Inst) override;
   void lowerInsertElement(const InstInsertElement *Inst) override;
   void lowerLoad(const InstLoad *Inst) override;
@@ -160,6 +162,7 @@ protected:
   Variable *legalizeToReg(Operand *From, int32_t RegNum = Variable::NoRegister);
   OperandARM32Mem *formMemoryOperand(Operand *Ptr, Type Ty);
 
+  Variable64On32 *makeI64RegPair();
   Variable *makeReg(Type Ty, int32_t RegNum = Variable::NoRegister);
   static Type stackSlotType();
   Variable *copyToReg(Operand *Src, int32_t RegNum = Variable::NoRegister);
@@ -299,6 +302,7 @@ protected:
       Context.insert(InstFakeDef::create(Func, Instr->getDestHi()));
     }
   }
+
   void _mov_redefined(Variable *Dest, Operand *Src0,
                       CondARM32::Cond Pred = CondARM32::AL) {
     auto *Instr = InstARM32Mov::create(Func, Dest, Src0, Pred);
@@ -385,11 +389,17 @@ protected:
   }
   void _strex(Variable *Dest, Variable *Value, OperandARM32Mem *Addr,
               CondARM32::Cond Pred = CondARM32::AL) {
+    // strex requires Dest to be a register other than Value or Addr. This
+    // restriction is cleanly represented by adding an "early" definition of
+    // Dest (or a latter use of all the sources.)
+    Context.insert(InstFakeDef::create(Func, Dest));
     if (auto *Value64 = llvm::dyn_cast<Variable64On32>(Value)) {
       Context.insert(InstFakeUse::create(Func, Value64->getLo()));
       Context.insert(InstFakeUse::create(Func, Value64->getHi()));
     }
-    Context.insert(InstARM32Strex::create(Func, Dest, Value, Addr, Pred));
+    auto *Instr = InstARM32Strex::create(Func, Dest, Value, Addr, Pred);
+    Context.insert(Instr);
+    Instr->setDestRedefined();
   }
   void _sub(Variable *Dest, Variable *Src0, Operand *Src1,
             CondARM32::Cond Pred = CondARM32::AL) {

tests_lit/llvm2ice_tests/nacl-atomic-intrinsics.ll

@@ -8,6 +8,21 @@
 ; RUN: %p2i -i %s --filetype=obj --disassemble --args -Om1 \
 ; RUN:   | FileCheck %s
 
+; RUN: %if --need=allow_dump --need=target_ARM32 --command %p2i --filetype=asm \
+; RUN:   --target arm32 -i %s --args -O2 --skip-unimplemented \
+; RUN:   | %if --need=allow_dump --need=target_ARM32 --command FileCheck %s \
+; RUN:   --check-prefix=ARM32
+
+; RUN: %if --need=allow_dump --need=target_ARM32 --command %p2i --filetype=asm \
+; RUN:   --target arm32 -i %s --args -O2 --skip-unimplemented \
+; RUN:   | %if --need=allow_dump --need=target_ARM32 --command FileCheck %s \
+; RUN:   --check-prefix=ARM32O2
+
+; RUN: %if --need=allow_dump --need=target_ARM32 --command %p2i --filetype=asm \
+; RUN:   --target arm32 -i %s --args -Om1 --skip-unimplemented \
+; RUN:   | %if --need=allow_dump --need=target_ARM32 --command FileCheck %s \
+; RUN:   --check-prefix=ARM32
+
 declare i8 @llvm.nacl.atomic.load.i8(i8*, i32)
 declare i16 @llvm.nacl.atomic.load.i16(i16*, i32)
 declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
@@ -55,6 +70,9 @@ entry:
 ; CHECK-LABEL: test_atomic_load_8
 ; CHECK: mov {{.*}},DWORD
 ; CHECK: mov {{.*}},BYTE
+; ARM32-LABEL: test_atomic_load_8
+; ARM32: ldrb r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i32 @test_atomic_load_16(i32 %iptr) {
 entry:
@@ -67,6 +85,9 @@ entry:
 ; CHECK-LABEL: test_atomic_load_16
 ; CHECK: mov {{.*}},DWORD
 ; CHECK: mov {{.*}},WORD
+; ARM32-LABEL: test_atomic_load_16
+; ARM32: ldrh r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i32 @test_atomic_load_32(i32 %iptr) {
 entry:
@@ -77,6 +98,9 @@ entry:
 ; CHECK-LABEL: test_atomic_load_32
 ; CHECK: mov {{.*}},DWORD
 ; CHECK: mov {{.*}},DWORD
+; ARM32-LABEL: test_atomic_load_32
+; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i64 @test_atomic_load_64(i32 %iptr) {
 entry:
@@ -87,6 +111,9 @@ entry:
 ; CHECK-LABEL: test_atomic_load_64
 ; CHECK: movq x{{.*}},QWORD
 ; CHECK: movq QWORD {{.*}},x{{.*}}
+; ARM32-LABEL: test_atomic_load_64
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i32 @test_atomic_load_32_with_arith(i32 %iptr) {
 entry:
@@ -106,6 +133,9 @@ next:
 ; O2-LABEL: test_atomic_load_32_with_arith
 ; O2: mov {{.*}},DWORD
 ; O2: sub {{.*}},DWORD
+; ARM32-LABEL: test_atomic_load_32_with_arith
+; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i32 @test_atomic_load_32_ignored(i32 %iptr) {
 entry:
@@ -119,6 +149,9 @@ entry:
 ; O2-LABEL: test_atomic_load_32_ignored
 ; O2: mov {{.*}},DWORD
 ; O2: mov {{.*}},DWORD
+; ARM32-LABEL: test_atomic_load_32_ignored
+; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define i64 @test_atomic_load_64_ignored(i32 %iptr) {
 entry:
@@ -129,6 +162,9 @@ entry:
 ; CHECK-LABEL: test_atomic_load_64_ignored
 ; CHECK: movq x{{.*}},QWORD
 ; CHECK: movq QWORD {{.*}},x{{.*}}
+; ARM32-LABEL: test_atomic_load_64_ignored
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 ;;; Store
 
@@ -142,6 +178,10 @@ entry:
 ; CHECK-LABEL: test_atomic_store_8
 ; CHECK: mov BYTE
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_store_8
+; ARM32: dmb
+; ARM32: strb r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define void @test_atomic_store_16(i32 %iptr, i32 %v) {
 entry:
@@ -153,6 +193,10 @@ entry:
 ; CHECK-LABEL: test_atomic_store_16
 ; CHECK: mov WORD
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_store_16
+; ARM32: dmb
+; ARM32: strh r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define void @test_atomic_store_32(i32 %iptr, i32 %v) {
 entry:
@@ -163,6 +207,10 @@ entry:
 ; CHECK-LABEL: test_atomic_store_32
 ; CHECK: mov DWORD
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_store_32
+; ARM32: dmb
+; ARM32: str r{{[0-9]+}}, [r{{[0-9]+}}
+; ARM32: dmb
 
 define void @test_atomic_store_64(i32 %iptr, i64 %v) {
 entry:
@@ -174,6 +222,13 @@ entry:
 ; CHECK: movq x{{.*}},QWORD
 ; CHECK: movq QWORD {{.*}},x{{.*}}
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_store_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [[MEM:.*]]
+; ARM32: strexd [[S:r[0-9]+]], r{{[0-9]+}}, r{{[0-9]+}}, [[MEM]]
+; ARM32: cmp [[S]], #0
+; ARM32: bne
+; ARM32: dmb
 
 define void @test_atomic_store_64_const(i32 %iptr) {
 entry:
@@ -187,7 +242,17 @@ entry:
 ; CHECK: movq x{{.*}},QWORD
 ; CHECK: movq QWORD {{.*}},x{{.*}}
 ; CHECK: mfence
-
+; ARM32-LABEL: test_atomic_store_64_const
+; ARM32: dmb
+; ARM32: movw [[T0:r[0-9]+]], #12274
+; ARM32: movt [[T0]], #29646
+; ARM32: movw r{{[0-9]+}}, #2874
+; ARM32: .L[[RETRY:.*]]:
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [[MEM:.*]]
+; ARM32: strexd [[S:r[0-9]+]], r{{[0-9]+}}, r{{[0-9]+}}, [[MEM]]
+; ARM32: cmp [[S]], #0
+; ARM32: bne .L[[RETRY]]
+; ARM32: dmb
 
 ;;; RMW
 
@@ -205,6 +270,13 @@ entry:
 ; CHECK-LABEL: test_atomic_rmw_add_8
 ; CHECK: lock xadd BYTE {{.*}},[[REG:.*]]
 ; CHECK: {{mov|movzx}} {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_add_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: add
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_add_16(i32 %iptr, i32 %v) {
 entry:
@@ -217,6 +289,13 @@ entry:
 ; CHECK-LABEL: test_atomic_rmw_add_16
 ; CHECK: lock xadd WORD {{.*}},[[REG:.*]]
 ; CHECK: {{mov|movzx}} {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_add_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: add
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_add_32(i32 %iptr, i32 %v) {
 entry:
@@ -227,6 +306,13 @@ entry:
 ; CHECK-LABEL: test_atomic_rmw_add_32
 ; CHECK: lock xadd DWORD {{.*}},[[REG:.*]]
 ; CHECK: mov {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_add_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: add
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_add_64(i32 %iptr, i64 %v) {
 entry:
@@ -249,6 +335,14 @@ entry:
 ; about rejecting eb* and ed*.)
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK: jne [[LABEL]]
+; ARM32-LABEL: test_atomic_rmw_add_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: adds
+; ARM32-NEXT: adc
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define i64 @test_atomic_rmw_add_64_global(i64 %v) {
@@ -258,6 +352,14 @@ entry:
   ret i64 %a
 }
 ; CHECK-LABEL: test_atomic_rmw_add_64_global
+; ARM32-LABEL: test_atomic_rmw_add_64_global
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: adds
+; ARM32-NEXT: adc
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 ; Test with some more register pressure. When we have an alloca, ebp is
 ; used to manage the stack frame, so it cannot be used as a register either.
@@ -291,6 +393,14 @@ eblock:
 ; That pretty much leaves esi, or edi as the only viable registers.
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{[ds]}}i]
 ; CHECK: call {{.*}} R_{{.*}} use_ptr
+; ARM32-LABEL: test_atomic_rmw_add_64_alloca
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: adds
+; ARM32-NEXT: adc
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_add_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -302,6 +412,13 @@ entry:
 ; tells us that the destination variable is dead.
 ; CHECK-LABEL: test_atomic_rmw_add_32_ignored
 ; CHECK: lock xadd DWORD {{.*}},[[REG:.*]]
+; ARM32-LABEL: test_atomic_rmw_add_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: add
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ; Atomic RMW 64 needs to be expanded into its own loop.
 ; Make sure that works w/ non-trivial function bodies.
@@ -333,6 +450,15 @@ err:
 ; CHECK: adc ecx,{{.*e.[^x]}}
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}+0x0]
 ; CHECK: jne [[LABEL]]
+; ARM32-LABEL: test_atomic_rmw_add_64_loop
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: adds
+; ARM32-NEXT: adc
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
+; ARM32: b
 
 ;; sub
 
@@ -348,6 +474,13 @@ entry:
 ; CHECK: neg [[REG:.*]]
 ; CHECK: lock xadd BYTE {{.*}},[[REG]]
 ; CHECK: {{mov|movzx}} {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_sub_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: sub
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_sub_16(i32 %iptr, i32 %v) {
 entry:
@@ -361,6 +494,13 @@ entry:
 ; CHECK: neg [[REG:.*]]
 ; CHECK: lock xadd WORD {{.*}},[[REG]]
 ; CHECK: {{mov|movzx}} {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_sub_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: sub
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_sub_32(i32 %iptr, i32 %v) {
 entry:
@@ -372,6 +512,13 @@ entry:
 ; CHECK: neg [[REG:.*]]
 ; CHECK: lock xadd DWORD {{.*}},[[REG]]
 ; CHECK: mov {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_sub_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: sub
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_sub_64(i32 %iptr, i64 %v) {
 entry:
@@ -389,7 +536,14 @@ entry:
 ; CHECK: sbb ecx,{{.*e.[^x]}}
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK: jne [[LABEL]]
-
+; ARM32-LABEL: test_atomic_rmw_sub_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: subs
+; ARM32-NEXT: sbc
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_sub_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -401,6 +555,13 @@ entry:
 ; CHECK-LABEL: test_atomic_rmw_sub_32_ignored
 ; CHECK: neg [[REG:.*]]
 ; CHECK: lock xadd DWORD {{.*}},[[REG]]
+; ARM32-LABEL: test_atomic_rmw_sub_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: sub
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ;; or
 
@@ -419,6 +580,13 @@ entry:
 ; CHECK: or [[REG:[^a].]]
 ; CHECK: lock cmpxchg BYTE PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_or_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: orr
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define i32 @test_atomic_rmw_or_8_global(i32 %v) {
@@ -430,6 +598,15 @@ entry:
   ret i32 %a_ext
 }
 ; CHECK-LABEL: test_atomic_rmw_or_8_global
+; ARM32-LABEL: test_atomic_rmw_or_8_global
+; ARM32: movw [[PTR:r[0-9]+]], #:lower16:Global8
+; ARM32: movt [[PTR]], #:upper16:Global8
+; ARM32: dmb
+; ARM32: ldrexb r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: orr
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_or_16(i32 %iptr, i32 %v) {
 entry:
@@ -444,6 +621,13 @@ entry:
 ; CHECK: or [[REG:[^a].]]
 ; CHECK: lock cmpxchg WORD PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_or_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: orr
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define i32 @test_atomic_rmw_or_16_global(i32 %v) {
@@ -455,6 +639,15 @@ entry:
   ret i32 %a_ext
 }
 ; CHECK-LABEL: test_atomic_rmw_or_16_global
+; ARM32-LABEL: test_atomic_rmw_or_16_global
+; ARM32: movw [[PTR:r[0-9]+]], #:lower16:Global16
+; ARM32: movt [[PTR]], #:upper16:Global16
+; ARM32: dmb
+; ARM32: ldrexh r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: orr
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_or_32(i32 %iptr, i32 %v) {
 entry:
@@ -467,6 +660,13 @@ entry:
 ; CHECK: or [[REG:e[^a].]]
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_or_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: orr
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define i32 @test_atomic_rmw_or_32_global(i32 %v) {
@@ -476,6 +676,15 @@ entry:
   ret i32 %a
 }
 ; CHECK-LABEL: test_atomic_rmw_or_32_global
+; ARM32-LABEL: test_atomic_rmw_or_32_global
+; ARM32: movw [[PTR:r[0-9]+]], #:lower16:Global32
+; ARM32: movt [[PTR]], #:upper16:Global32
+; ARM32: dmb
+; ARM32: ldrex r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: orr
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_or_64(i32 %iptr, i64 %v) {
 entry:
@@ -493,6 +702,14 @@ entry:
 ; CHECK: or ecx,{{.*e.[^x]}}
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK: jne [[LABEL]]
+; ARM32-LABEL: test_atomic_rmw_or_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: orr
+; ARM32-NEXT: orr
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_or_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -508,6 +725,13 @@ entry:
 ; CHECK: or [[REG:e[^a].]]
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_or_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: orr
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ;; and
 
@@ -524,6 +748,13 @@ entry:
 ; CHECK: and [[REG:[^a].]]
 ; CHECK: lock cmpxchg BYTE PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_and_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: and
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_and_16(i32 %iptr, i32 %v) {
 entry:
@@ -538,6 +769,13 @@ entry:
 ; CHECK: and
 ; CHECK: lock cmpxchg WORD PTR [e{{[^a].}}]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_and_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: and
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_and_32(i32 %iptr, i32 %v) {
 entry:
@@ -550,6 +788,13 @@ entry:
 ; CHECK: and
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_and_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: and
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_and_64(i32 %iptr, i64 %v) {
 entry:
@@ -567,6 +812,14 @@ entry:
 ; CHECK: and ecx,{{.*e.[^x]}}
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK: jne [[LABEL]]
+; ARM32-LABEL: test_atomic_rmw_and_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: and
+; ARM32-NEXT: and
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_and_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -580,6 +833,13 @@ entry:
 ; CHECK: and
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_and_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: and
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ;; xor
 
@@ -596,6 +856,13 @@ entry:
 ; CHECK: xor [[REG:[^a].]]
 ; CHECK: lock cmpxchg BYTE PTR [e{{[^a].}}],[[REG]]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_xor_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: eor
+; ARM32: strexb
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xor_16(i32 %iptr, i32 %v) {
 entry:
@@ -610,7 +877,13 @@ entry:
 ; CHECK: xor
 ; CHECK: lock cmpxchg WORD PTR [e{{[^a].}}]
 ; CHECK: jne
-
+; ARM32-LABEL: test_atomic_rmw_xor_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: eor
+; ARM32: strexh
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xor_32(i32 %iptr, i32 %v) {
 entry:
@@ -623,6 +896,13 @@ entry:
 ; CHECK: xor
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_xor_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: eor
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_xor_64(i32 %iptr, i64 %v) {
 entry:
@@ -640,6 +920,14 @@ entry:
 ; CHECK: or ecx,{{.*e.[^x]}}
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_xor_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: eor
+; ARM32-NEXT: eor
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xor_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -652,6 +940,13 @@ entry:
 ; CHECK: xor
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}]
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_xor_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: eor
+; ARM32: strex
+; ARM32: bne
+; ARM32: dmb
 
 ;; exchange
 
@@ -665,6 +960,13 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_rmw_xchg_8
 ; CHECK: xchg BYTE PTR {{.*}},[[REG:.*]]
+; ARM32-LABEL: test_atomic_rmw_xchg_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: strexb
+; ARM32: cmp
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xchg_16(i32 %iptr, i32 %v) {
 entry:
@@ -676,6 +978,13 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_rmw_xchg_16
 ; CHECK: xchg WORD PTR {{.*}},[[REG:.*]]
+; ARM32-LABEL: test_atomic_rmw_xchg_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: strexh
+; ARM32: cmp
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xchg_32(i32 %iptr, i32 %v) {
 entry:
@@ -685,6 +994,13 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_rmw_xchg_32
 ; CHECK: xchg DWORD PTR {{.*}},[[REG:.*]]
+; ARM32-LABEL: test_atomic_rmw_xchg_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: strex
+; ARM32: cmp
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_rmw_xchg_64(i32 %iptr, i64 %v) {
 entry:
@@ -700,6 +1016,13 @@ entry:
 ; CHECK-DAG: mov ebx
 ; CHECK: lock cmpxchg8b QWORD PTR [{{e.[^x]}}
 ; CHECK: jne
+; ARM32-LABEL: test_atomic_rmw_xchg_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: cmp
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_rmw_xchg_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -711,6 +1034,13 @@ entry:
 ; used to do an atomic store.
 ; CHECK-LABEL: test_atomic_rmw_xchg_32_ignored
 ; CHECK: xchg DWORD PTR {{.*}},[[REG:.*]]
+; ARM32-LABEL: test_atomic_rmw_xchg_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: strex
+; ARM32: cmp
+; ARM32: bne
+; ARM32: dmb
 
 ;;;; Cmpxchg
 
@@ -729,6 +1059,15 @@ entry:
 ; Need to check that eax isn't used as the address register or the desired.
 ; since it is already used as the *expected* register.
 ; CHECK: lock cmpxchg BYTE PTR [e{{[^a].}}],{{[^a]}}l
+; ARM32-LABEL: test_atomic_cmpxchg_8
+; ARM32: dmb
+; ARM32: ldrexb
+; ARM32: cmp
+; ARM32: strexbeq
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_cmpxchg_16(i32 %iptr, i32 %expected, i32 %desired) {
 entry:
@@ -743,6 +1082,15 @@ entry:
 ; CHECK-LABEL: test_atomic_cmpxchg_16
 ; CHECK: mov eax,{{.*}}
 ; CHECK: lock cmpxchg WORD PTR [e{{[^a].}}],{{[^a]}}x
+; ARM32-LABEL: test_atomic_cmpxchg_16
+; ARM32: dmb
+; ARM32: ldrexh
+; ARM32: cmp
+; ARM32: strexheq
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_cmpxchg_32(i32 %iptr, i32 %expected, i32 %desired) {
 entry:
@@ -754,6 +1102,15 @@ entry:
 ; CHECK-LABEL: test_atomic_cmpxchg_32
 ; CHECK: mov eax,{{.*}}
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}],e{{[^a]}}
+; ARM32-LABEL: test_atomic_cmpxchg_32
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: cmp
+; ARM32: strexeq
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_cmpxchg_64(i32 %iptr, i64 %expected, i64 %desired) {
 entry:
@@ -772,6 +1129,17 @@ entry:
 ; edx and eax are already the return registers, so they don't actually
 ; need to be reshuffled via movs. The next test stores the result
 ; somewhere, so in that case they do need to be mov'ed.
+; ARM32-LABEL: test_atomic_cmpxchg_64
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: cmp
+; ARM32-NEXT: cmpeq
+; ARM32: strexdeq r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: {{str|mov}}ne
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_cmpxchg_64_undef(i32 %iptr, i64 %desired) {
 entry:
@@ -782,6 +1150,19 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_cmpxchg_64_undef
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}+0x0]
+; ARM32-LABEL: test_atomic_cmpxchg_64_undef
+; ARM32: mov r{{[0-9]+}}, #0
+; ARM32: mov r{{[0-9]+}}, #0
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: cmp
+; ARM32-NEXT: cmpeq
+; ARM32: strexdeq r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: {{str|mov}}ne
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 ; Test a case where %old really does need to be copied out of edx:eax.
 define void @test_atomic_cmpxchg_64_store(i32 %ret_iptr, i32 %iptr, i64 %expected, i64 %desired) {
@@ -802,6 +1183,19 @@ entry:
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}
 ; CHECK-DAG: mov {{.*}},edx
 ; CHECK-DAG: mov {{.*}},eax
+; ARM32-LABEL: test_atomic_cmpxchg_64_store
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: cmp
+; ARM32-NEXT: cmpeq
+; ARM32: strexdeq r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: {{str|mov}}ne
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
+; ARM32: str
+; ARM32: str
 
 ; Test with some more register pressure. When we have an alloca, ebp is
 ; used to manage the stack frame, so it cannot be used as a register either.
@@ -834,6 +1228,17 @@ eblock:
 ; That pretty much leaves esi, or edi as the only viable registers.
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{[ds]}}i]
 ; CHECK: call {{.*}} R_{{.*}} use_ptr
+; ARM32-LABEL: test_atomic_cmpxchg_64_alloca
+; ARM32: dmb
+; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: cmp
+; ARM32-NEXT: cmpeq
+; ARM32: strexdeq r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32: {{str|mov}}ne
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i32 @test_atomic_cmpxchg_32_ignored(i32 %iptr, i32 %expected, i32 %desired) {
 entry:
@@ -845,6 +1250,15 @@ entry:
 ; CHECK-LABEL: test_atomic_cmpxchg_32_ignored
 ; CHECK: mov eax,{{.*}}
 ; CHECK: lock cmpxchg DWORD PTR [e{{[^a].}}]
+; ARM32-LABEL: test_atomic_cmpxchg_32_ignored
+; ARM32: dmb
+; ARM32: ldrex
+; ARM32: cmp
+; ARM32: strexeq
+; ARM32: {{str|mov}}ne
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 define i64 @test_atomic_cmpxchg_64_ignored(i32 %iptr, i64 %expected, i64 %desired) {
 entry:
@@ -860,6 +1274,17 @@ entry:
 ; CHECK-DAG: mov ecx
 ; CHECK-DAG: mov ebx
 ; CHECK: lock cmpxchg8b QWORD PTR [e{{.[^x]}}+0x0]
+; ARM32-LABEL: test_atomic_cmpxchg_64_ignored
+; ARM32: dmb
+; ARM32: ldrexd [[R0:r[0-9]+]], [[R1:r[0-9]+]], {{[[]}}[[PTR:r[0-9]+]]{{[]]}}
+; ARM32: cmp
+; ARM32-NEXT: cmpeq
+; ARM32: strexdeq r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, {{[[]}}[[PTR]]{{[]]}}
+; ARM32O2-NOT: {{str|mov}}ne [[R0]]
+; ARM32O2-NOT: {{str|mov}}ne [[R1]]
+; ARM32: cmpeq
+; ARM32: bne
+; ARM32: dmb
 
 ;;;; Fence and is-lock-free.
 
@@ -870,6 +1295,8 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_fence
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_fence
+; ARM32: dmb sy
 
 define void @test_atomic_fence_all() {
 entry:
@@ -878,6 +1305,8 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_fence_all
 ; CHECK: mfence
+; ARM32-LABEL: test_atomic_fence_all
+; ARM32: dmb sy
 
 define i32 @test_atomic_is_lock_free(i32 %iptr) {
 entry:
@@ -888,6 +1317,8 @@ entry:
 }
 ; CHECK-LABEL: test_atomic_is_lock_free
 ; CHECK: mov {{.*}},0x1
+; ARM32-LABEL: test_atomic_is_lock_free
+; ARM32: movw {{.*}}, #1
 
 define i32 @test_not_lock_free(i32 %iptr) {
 entry:
@@ -898,6 +1329,8 @@ entry:
 }
 ; CHECK-LABEL: test_not_lock_free
 ; CHECK: mov {{.*}},0x0
+; ARM32-LABEL: test_not_lock_free
+; ARM32: mov {{.*}}, #0
 
 define i32 @test_atomic_is_lock_free_ignored(i32 %iptr) {
 entry:
@@ -911,6 +1344,9 @@ entry:
 ; O2-LABEL: test_atomic_is_lock_free_ignored
 ; O2-NOT: mov {{.*}}, 1
 ; O2: mov {{.*}},0x0
+; ARM32O2-LABEL: test_atomic_is_lock_free_ignored
+; ARM32O2-NOT: mov {{.*}}, #1
+; ARM32O2: mov {{.*}}, #0
 
 ; TODO(jvoung): at some point we can take advantage of the
 ; fact that nacl.atomic.is.lock.free will resolve to a constant