[Subzero][MIPS] Implements atomic intrinsics for MIPS32

crosstest/test_sync_atomic_main.cpp

@@ -160,14 +160,14 @@ template <typename Type> struct ThreadData {
 };
 
 template <typename Type> void *threadWrapper(void *Data) {
-#ifdef ARM32
+#if defined(ARM32) || defined(MIPS32)
   // Given that most of times these crosstests for ARM are run under qemu, we
   // set a lower NumReps to allow crosstests to complete within a reasonable
   // amount of time.
   static const size_t NumReps = 1000;
-#else  // ARM32
+#else  // ARM32 || MIPS32
   static const size_t NumReps = 8000;
-#endif // ARM32
+#endif // ARM32 || MIPS32
 
   ThreadData<Type> *TData = reinterpret_cast<ThreadData<Type> *>(Data);
   for (size_t i = 0; i < NumReps; ++i) {

src/IceAssemblerMIPS32.cpp

@@ -696,6 +696,12 @@ void AssemblerMIPS32::ldc1(const Operand *OpRt, const Operand *OpBase,
   emitInst(Opcode);
 }
 
+void AssemblerMIPS32::ll(const Operand *OpRt, const Operand *OpBase,
+                         const uint32_t Offset) {
+  static constexpr IValueT Opcode = 0xC0000000;
+  emitRtRsImm16(Opcode, OpRt, OpBase, Offset, "ll");
+}
+
 void AssemblerMIPS32::lw(const Operand *OpRt, const Operand *OpBase,
                          const uint32_t Offset) {
   switch (OpRt->getType()) {
@@ -959,6 +965,12 @@ void AssemblerMIPS32::ret(void) {
   nop(); // delay slot
 }
 
+void AssemblerMIPS32::sc(const Operand *OpRt, const Operand *OpBase,
+                         const uint32_t Offset) {
+  static constexpr IValueT Opcode = 0xE0000000;
+  emitRtRsImm16(Opcode, OpRt, OpBase, Offset, "sc");
+}
+
 void AssemblerMIPS32::sll(const Operand *OpRd, const Operand *OpRt,
                           const uint32_t Sa) {
   static constexpr IValueT Opcode = 0x00000000;
@@ -1122,6 +1134,11 @@ void AssemblerMIPS32::swc1(const Operand *OpRt, const Operand *OpBase,
   emitInst(Opcode);
 }
 
+void AssemblerMIPS32::sync() {
+  static constexpr IValueT Opcode = 0x0000000f;
+  emitInst(Opcode);
+}
+
 void AssemblerMIPS32::teq(const Operand *OpRs, const Operand *OpRt,
                           const uint32_t TrapCode) {
   IValueT Opcode = 0x00000034;

src/IceAssemblerMIPS32.h

@@ -195,6 +195,8 @@ public:
   void ldc1(const Operand *OpRt, const Operand *OpBase, const Operand *OpOff,
             const RelocOp Reloc);
 
+  void ll(const Operand *OpRt, const Operand *OpBase, const uint32_t Offset);
+
   void lw(const Operand *OpRt, const Operand *OpBase, const uint32_t Offset);
 
   void lwc1(const Operand *OpRt, const Operand *OpBase, const Operand *OpOff,
@@ -252,6 +254,8 @@ public:
 
   void ret(void);
 
+  void sc(const Operand *OpRt, const Operand *OpBase, const uint32_t Offset);
+
   void sll(const Operand *OpRd, const Operand *OpRt, const uint32_t Sa);
 
   void sllv(const Operand *OpRd, const Operand *OpRt, const Operand *OpRs);
@@ -290,6 +294,8 @@ public:
   void swc1(const Operand *OpRt, const Operand *OpBase, const Operand *OpOff,
             const RelocOp Reloc);
 
+  void sync();
+
   void teq(const Operand *OpRs, const Operand *OpRt, const uint32_t TrapCode);
 
   void trunc_l_d(const Operand *OpFd, const Operand *OpFs);

src/IceInstMIPS32.cpp

@@ -97,6 +97,7 @@ template <> const char *InstMIPS32Div_s::Opcode = "div.s";
 template <> const char *InstMIPS32Divu::Opcode = "divu";
 template <> const char *InstMIPS32La::Opcode = "la";
 template <> const char *InstMIPS32Ldc1::Opcode = "ldc1";
+template <> const char *InstMIPS32Ll::Opcode = "ll";
 template <> const char *InstMIPS32Lui::Opcode = "lui";
 template <> const char *InstMIPS32Lw::Opcode = "lw";
 template <> const char *InstMIPS32Lwc1::Opcode = "lwc1";
@@ -124,6 +125,7 @@ template <> const char *InstMIPS32Multu::Opcode = "multu";
 template <> const char *InstMIPS32Nor::Opcode = "nor";
 template <> const char *InstMIPS32Or::Opcode = "or";
 template <> const char *InstMIPS32Ori::Opcode = "ori";
+template <> const char *InstMIPS32Sc::Opcode = "sc";
 template <> const char *InstMIPS32Sdc1::Opcode = "sdc1";
 template <> const char *InstMIPS32Sll::Opcode = "sll";
 template <> const char *InstMIPS32Sllv::Opcode = "sllv";
@@ -143,6 +145,7 @@ template <> const char *InstMIPS32Sub_s::Opcode = "sub.s";
 template <> const char *InstMIPS32Subu::Opcode = "subu";
 template <> const char *InstMIPS32Sw::Opcode = "sw";
 template <> const char *InstMIPS32Swc1::Opcode = "swc1";
+const char *InstMIPS32Sync::Opcode = "sync";
 template <> const char *InstMIPS32Teq::Opcode = "teq";
 template <> const char *InstMIPS32Trunc_l_d::Opcode = "trunc.l.d";
 template <> const char *InstMIPS32Trunc_l_s::Opcode = "trunc.l.s";
@@ -429,8 +432,12 @@ void InstMIPS32Br::emitIAS(const Cfg *Func) const {
   auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
   if (Label != nullptr) {
     // Intra-block branches are of kind bcc
-    Asm->bcc(Predicate, getSrc(0), getSrc(1),
-             Asm->getOrCreateLocalLabel(Label->getNumber()));
+    if (isUnconditionalBranch()) {
+      Asm->b(Asm->getOrCreateLocalLabel(Label->getNumber()));
+    } else {
+      Asm->bcc(Predicate, getSrc(0), getSrc(1),
+               Asm->getOrCreateLocalLabel(Label->getNumber()));
+    }
   } else if (isUnconditionalBranch()) {
     Asm->b(Asm->getOrCreateCfgNodeLabel(getTargetFalse()->getIndex()));
   } else {
@@ -465,10 +472,14 @@ void InstMIPS32Br::emit(const Cfg *Func) const {
   Str << "\t"
          "b" << InstMIPS32CondAttributes[Predicate].EmitString << "\t";
   if (Label != nullptr) {
-    getSrc(0)->emit(Func);
-    Str << ", ";
-    getSrc(1)->emit(Func);
-    Str << ", " << Label->getLabelName();
+    if (isUnconditionalBranch()) {
+      Str << Label->getLabelName();
+    } else {
+      getSrc(0)->emit(Func);
+      Str << ", ";
+      getSrc(1)->emit(Func);
+      Str << ", " << Label->getLabelName();
+    }
   } else {
     if (isUnconditionalBranch()) {
       Str << getTargetFalse()->getAsmName();
@@ -513,10 +524,14 @@ void InstMIPS32Br::dump(const Cfg *Func) const {
          "b" << InstMIPS32CondAttributes[Predicate].EmitString << "\t";
 
   if (Label != nullptr) {
-    getSrc(0)->dump(Func);
-    Str << ", ";
-    getSrc(1)->dump(Func);
-    Str << ", " << Label->getLabelName();
+    if (isUnconditionalBranch()) {
+      Str << Label->getLabelName();
+    } else {
+      getSrc(0)->dump(Func);
+      Str << ", ";
+      getSrc(1)->dump(Func);
+      Str << ", " << Label->getLabelName();
+    }
   } else {
     if (isUnconditionalBranch()) {
       Str << getTargetFalse()->getAsmName();
@@ -886,6 +901,14 @@ template <> void InstMIPS32Ldc1::emitIAS(const Cfg *Func) const {
   Asm->ldc1(getDest(), Mem->getBase(), Mem->getOffset(), Reloc);
 }
 
+template <> void InstMIPS32Ll::emitIAS(const Cfg *Func) const {
+  auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
+  auto *Mem = llvm::dyn_cast<OperandMIPS32Mem>(getSrc(0));
+  ConstantInteger32 *Offset = llvm::cast<ConstantInteger32>(Mem->getOffset());
+  uint32_t Imm = static_cast<uint32_t>(Offset->getValue());
+  Asm->ll(getDest(), Mem->getBase(), Imm);
+}
+
 template <> void InstMIPS32Lw::emitIAS(const Cfg *Func) const {
   auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
   auto *Mem = llvm::dyn_cast<OperandMIPS32Mem>(getSrc(0));
@@ -1067,6 +1090,14 @@ template <> void InstMIPS32Ori::emitIAS(const Cfg *Func) const {
   Asm->ori(getDest(), getSrc(0), Imm);
 }
 
+template <> void InstMIPS32Sc::emitIAS(const Cfg *Func) const {
+  auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
+  auto *Mem = llvm::dyn_cast<OperandMIPS32Mem>(getSrc(1));
+  ConstantInteger32 *Offset = llvm::cast<ConstantInteger32>(Mem->getOffset());
+  uint32_t Imm = static_cast<uint32_t>(Offset->getValue());
+  Asm->sc(getSrc(0), Mem->getBase(), Imm);
+}
+
 template <> void InstMIPS32Sll::emitIAS(const Cfg *Func) const {
   auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
   Asm->sll(getDest(), getSrc(0), Imm);
@@ -1162,6 +1193,11 @@ template <> void InstMIPS32Swc1::emitIAS(const Cfg *Func) const {
   Asm->swc1(getSrc(0), Mem->getBase(), Mem->getOffset(), Reloc);
 }
 
+void InstMIPS32Sync::emitIAS(const Cfg *Func) const {
+  auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
+  Asm->sync();
+}
+
 template <> void InstMIPS32Teq::emitIAS(const Cfg *Func) const {
   auto *Asm = Func->getAssembler<MIPS32::AssemblerMIPS32>();
   Asm->teq(getSrc(0), getSrc(1), getTrapCode());

src/IceInstMIPS32.h

@@ -224,6 +224,7 @@ public:
     La,
     Label,
     Ldc1,
+    Ll,
     Lui,
     Lw,
     Lwc1,
@@ -253,6 +254,7 @@ public:
     Or,
     Ori,
     Ret,
+    Sc,
     Sdc1,
     Sll,
     Sllv,
@@ -272,6 +274,7 @@ public:
     Subu,
     Sw,
     Swc1,
+    Sync,
     Teq,
     Trunc_l_d,
     Trunc_l_s,
@@ -598,35 +601,40 @@ public:
       return;
     Ostream &Str = Func->getContext()->getStrEmit();
     const Type Ty = getDest()->getType();
-    switch (Ty) {
-    case IceType_i1:
-    case IceType_i8:
-      Str << "\t"
-          << "lb"
-          << "\t";
-      break;
-    case IceType_i16:
-      Str << "\t"
-          << "lh"
-          << "\t";
-      break;
-    case IceType_i32:
-      Str << "\t"
-          << "lw"
-          << "\t";
-      break;
-    case IceType_f32:
-      Str << "\t"
-          << "lwc1"
-          << "\t";
-      break;
-    case IceType_f64:
-      Str << "\t"
-          << "ldc1"
-          << "\t";
-      break;
-    default:
-      llvm_unreachable("InstMIPS32Load unknown type");
+
+    if (getKind() == static_cast<InstKind>(Ll)) {
+      Str << "\t" << Opcode << "\t";
+    } else {
+      switch (Ty) {
+      case IceType_i1:
+      case IceType_i8:
+        Str << "\t"
+               "lb"
+               "\t";
+        break;
+      case IceType_i16:
+        Str << "\t"
+               "lh"
+               "\t";
+        break;
+      case IceType_i32:
+        Str << "\t"
+               "lw"
+               "\t";
+        break;
+      case IceType_f32:
+        Str << "\t"
+               "lwc1"
+               "\t";
+        break;
+      case IceType_f64:
+        Str << "\t"
+               "ldc1"
+               "\t";
+        break;
+      default:
+        llvm_unreachable("InstMIPS32Load unknown type");
+      }
     }
     getDest()->emit(Func);
     Str << ", ";
@@ -683,35 +691,40 @@ public:
     Ostream &Str = Func->getContext()->getStrEmit();
     assert(getSrcSize() == 2);
     const Type Ty = getSrc(0)->getType();
-    switch (Ty) {
-    case IceType_i1:
-    case IceType_i8:
-      Str << "\t"
-          << "sb"
-          << "\t";
-      break;
-    case IceType_i16:
-      Str << "\t"
-          << "sh"
-          << "\t";
-      break;
-    case IceType_i32:
-      Str << "\t"
-          << "sw"
-          << "\t";
-      break;
-    case IceType_f32:
-      Str << "\t"
-          << "swc1"
-          << "\t";
-      break;
-    case IceType_f64:
-      Str << "\t"
-          << "sdc1"
-          << "\t";
-      break;
-    default:
-      llvm_unreachable("InstMIPS32Store unknown type");
+
+    if (getKind() == static_cast<InstKind>(Sc)) {
+      Str << "\t" << Opcode << "\t";
+    } else {
+      switch (Ty) {
+      case IceType_i1:
+      case IceType_i8:
+        Str << "\t"
+               "sb"
+               "\t";
+        break;
+      case IceType_i16:
+        Str << "\t"
+               "sh"
+               "\t";
+        break;
+      case IceType_i32:
+        Str << "\t"
+               "sw"
+               "\t";
+        break;
+      case IceType_f32:
+        Str << "\t"
+               "swc1"
+               "\t";
+        break;
+      case IceType_f64:
+        Str << "\t"
+               "sdc1"
+               "\t";
+        break;
+      default:
+        llvm_unreachable("InstMIPS32Store unknown type");
+      }
     }
     getSrc(0)->emit(Func);
     Str << ", ";
@@ -793,6 +806,13 @@ public:
         InstMIPS32Br(Func, NoCondTarget, Target, NoLabel, CondMIPS32::AL);
   }
 
+  static InstMIPS32Br *create(Cfg *Func, CfgNode *Target,
+                              const InstMIPS32Label *Label) {
+    constexpr CfgNode *NoCondTarget = nullptr;
+    return new (Func->allocate<InstMIPS32Br>())
+        InstMIPS32Br(Func, NoCondTarget, Target, Label, CondMIPS32::AL);
+  }
+
   /// Create a conditional branch to the false node.
   static InstMIPS32Br *create(Cfg *Func, CfgNode *TargetTrue,
                               CfgNode *TargetFalse, Operand *Src0,
@@ -921,6 +941,40 @@ private:
   static const char *Opcode;
 };
 
+class InstMIPS32Sync : public InstMIPS32 {
+  InstMIPS32Sync() = delete;
+  InstMIPS32Sync(const InstMIPS32Sync &) = delete;
+  InstMIPS32Sync &operator=(const InstMIPS32Sync &) = delete;
+
+public:
+  static InstMIPS32Sync *create(Cfg *Func) {
+    return new (Func->allocate<InstMIPS32Sync>()) InstMIPS32Sync(Func);
+  }
+
+  void emit(const Cfg *Func) const override {
+    if (!BuildDefs::dump())
+      return;
+    Ostream &Str = Func->getContext()->getStrEmit();
+    Str << "\t" << Opcode << "\t";
+  }
+
+  void dump(const Cfg *Func) const override {
+    if (!BuildDefs::dump())
+      return;
+    Func->getContext()->getStrDump() << Opcode << "\t";
+  }
+
+  static bool classof(const Inst *Inst) {
+    return isClassof(Inst, InstMIPS32::Sync);
+  }
+
+  void emitIAS(const Cfg *Func) const override;
+
+private:
+  InstMIPS32Sync(Cfg *Func) : InstMIPS32(Func, InstMIPS32::Sync, 0, nullptr) {}
+  static const char *Opcode;
+};
+
 // Trap
 template <InstMIPS32::InstKindMIPS32 K>
 class InstMIPS32Trap : public InstMIPS32 {
@@ -1175,6 +1229,7 @@ using InstMIPS32Div_s = InstMIPS32ThreeAddrFPR<InstMIPS32::Div_s>;
 using InstMIPS32Divu = InstMIPS32ThreeAddrGPR<InstMIPS32::Divu>;
 using InstMIPS32La = InstMIPS32UnaryopGPR<InstMIPS32::La>;
 using InstMIPS32Ldc1 = InstMIPS32Load<InstMIPS32::Ldc1>;
+using InstMIPS32Ll = InstMIPS32Load<InstMIPS32::Ll>;
 using InstMIPS32Lui = InstMIPS32UnaryopGPR<InstMIPS32::Lui>;
 using InstMIPS32Lw = InstMIPS32Load<InstMIPS32::Lw>;
 using InstMIPS32Lwc1 = InstMIPS32Load<InstMIPS32::Lwc1>;
@@ -1202,6 +1257,7 @@ using InstMIPS32Multu = InstMIPS32ThreeAddrGPR<InstMIPS32::Multu>;
 using InstMIPS32Nor = InstMIPS32ThreeAddrGPR<InstMIPS32::Nor>;
 using InstMIPS32Or = InstMIPS32ThreeAddrGPR<InstMIPS32::Or>;
 using InstMIPS32Ori = InstMIPS32Imm16<InstMIPS32::Ori>;
+using InstMIPS32Sc = InstMIPS32Store<InstMIPS32::Sc>;
 using InstMIPS32Sdc1 = InstMIPS32Store<InstMIPS32::Sdc1>;
 using InstMIPS32Sll = InstMIPS32Imm16<InstMIPS32::Sll>;
 using InstMIPS32Sllv = InstMIPS32ThreeAddrGPR<InstMIPS32::Sllv>;

src/IceTargetLoweringMIPS32.cpp

@@ -240,6 +240,14 @@ uint32_t TargetMIPS32::getCallStackArgumentsSizeBytes(const InstCall *Call) {
   return applyStackAlignment(OutArgsSizeBytes);
 }
 
+namespace {
+inline uint64_t getConstantMemoryOrder(Operand *Opnd) {
+  if (auto *Integer = llvm::dyn_cast<ConstantInteger32>(Opnd))
+    return Integer->getValue();
+  return Intrinsics::MemoryOrderInvalid;
+}
+}
+
 void TargetMIPS32::genTargetHelperCallFor(Inst *Instr) {
   constexpr bool NoTailCall = false;
   constexpr bool IsTargetHelperCall = true;
@@ -1053,7 +1061,7 @@ bool TargetMIPS32::CallingConv::argInReg(Type Ty, uint32_t ArgNo,
     }
     return argInGPR(Ty, Reg);
   }
-  UnimplementedError(getFlags());
+  llvm::report_fatal_error("argInReg: Invalid type.");
   return false;
 }
 
@@ -1062,7 +1070,7 @@ bool TargetMIPS32::CallingConv::argInGPR(Type Ty, RegNumT *Reg) {
 
   switch (Ty) {
   default: {
-    UnimplementedError(getFlags());
+    llvm::report_fatal_error("argInGPR: Invalid type.");
     return false;
   } break;
   case IceType_v4i1:
@@ -1147,7 +1155,7 @@ bool TargetMIPS32::CallingConv::argInVFP(Type Ty, RegNumT *Reg) {
 
   switch (Ty) {
   default: {
-    UnimplementedError(getFlags());
+    llvm::report_fatal_error("argInVFP: Invalid type.");
     return false;
   } break;
   case IceType_f32: {
@@ -2539,8 +2547,18 @@ void TargetMIPS32::lowerInt64Arithmetic(const InstArithmetic *Instr,
     _mov(DestLo, T1_Lo);
     return;
   }
-  default:
-    UnimplementedLoweringError(this, Instr);
+  case InstArithmetic::Fadd:
+  case InstArithmetic::Fsub:
+  case InstArithmetic::Fmul:
+  case InstArithmetic::Fdiv:
+  case InstArithmetic::Frem:
+    llvm::report_fatal_error("FP instruction with i64 type");
+    return;
+  case InstArithmetic::Udiv:
+  case InstArithmetic::Sdiv:
+  case InstArithmetic::Urem:
+  case InstArithmetic::Srem:
+    llvm::report_fatal_error("64-bit div and rem should have been prelowered");
     return;
   }
 }
@@ -2784,7 +2802,7 @@ void TargetMIPS32::lowerArithmetic(const InstArithmetic *Instr) {
     llvm::report_fatal_error("frem should have been prelowered.");
     break;
   }
-  UnimplementedLoweringError(this, Instr);
+  llvm::report_fatal_error("Unknown arithmetic operator");
 }
 
 void TargetMIPS32::lowerAssign(const InstAssign *Instr) {
@@ -3496,7 +3514,7 @@ void TargetMIPS32::lowerCast(const InstCast *Instr) {
         return;
       }
     }
-    UnimplementedLoweringError(this, Instr);
+    llvm::report_fatal_error("Destination is i64 in fp-to-i32");
     break;
   }
   case InstCast::Sitofp:
@@ -3529,7 +3547,7 @@ void TargetMIPS32::lowerCast(const InstCast *Instr) {
         return;
       }
     }
-    UnimplementedLoweringError(this, Instr);
+    llvm::report_fatal_error("Source is i64 in i32-to-fp");
     break;
   }
   case InstCast::Bitcast: {
@@ -3598,7 +3616,7 @@ void TargetMIPS32::lowerCast(const InstCast *Instr) {
       break;
     }
     default:
-      UnimplementedLoweringError(this, Instr);
+      llvm::report_fatal_error("Unexpected bitcast.");
     }
     break;
   }
@@ -3690,7 +3708,7 @@ void TargetMIPS32::lowerFcmp(const InstFcmp *Instr) {
 
   switch (Cond) {
   default: {
-    UnimplementedLoweringError(this, Instr);
+    llvm::report_fatal_error("Unhandled fp comparison.");
     return;
   }
   case InstFcmp::False: {
@@ -4235,36 +4253,430 @@ void TargetMIPS32::lowerInsertElement(const InstInsertElement *Instr) {
   llvm::report_fatal_error("InsertElement requires a constant index");
 }
 
+void TargetMIPS32::createArithInst(Intrinsics::AtomicRMWOperation Operation,
+                                   Variable *Dest, Variable *Src0,
+                                   Variable *Src1) {
+  switch (Operation) {
+  default:
+    llvm::report_fatal_error("Unknown AtomicRMW operation");
+  case Intrinsics::AtomicExchange:
+    llvm::report_fatal_error("Can't handle Atomic xchg operation");
+  case Intrinsics::AtomicAdd:
+    _addu(Dest, Src0, Src1);
+    break;
+  case Intrinsics::AtomicAnd:
+    _and(Dest, Src0, Src1);
+    break;
+  case Intrinsics::AtomicSub:
+    _subu(Dest, Src0, Src1);
+    break;
+  case Intrinsics::AtomicOr:
+    _or(Dest, Src0, Src1);
+    break;
+  case Intrinsics::AtomicXor:
+    _xor(Dest, Src0, Src1);
+    break;
+  }
+}
+
 void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
   Variable *Dest = Instr->getDest();
   Type DestTy = (Dest == nullptr) ? IceType_void : Dest->getType();
-  switch (Instr->getIntrinsicInfo().ID) {
-  case Intrinsics::AtomicCmpxchg: {
-    UnimplementedLoweringError(this, Instr);
+
+  Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID;
+  switch (ID) {
+  case Intrinsics::AtomicLoad: {
+    assert(isScalarIntegerType(DestTy));
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(1)))) {
+      Func->setError("Unexpected memory ordering for AtomicLoad");
+      return;
+    }
+    if (DestTy == IceType_i64) {
+      auto *Base = legalizeToReg(Instr->getArg(0));
+      auto *AddrLo = OperandMIPS32Mem::create(
+          Func, IceType_i32, Base,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
+      auto *AddrHi = OperandMIPS32Mem::create(
+          Func, IceType_i32, Base,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
+      auto *T_Lo = makeReg(IceType_i32);
+      auto *T_Hi = makeReg(IceType_i32);
+      auto *Dest64 = llvm::cast<Variable64On32>(Dest);
+      lowerLoad(InstLoad::create(Func, T_Lo, AddrLo, IceType_i32));
+      lowerLoad(InstLoad::create(Func, T_Hi, AddrHi, IceType_i32));
+      _sync();
+      _mov(Dest64->getLo(), T_Lo);
+      _mov(Dest64->getHi(), T_Hi);
+      // Adding a fake-use of T to ensure the atomic load is not removed if Dest
+      // is unused.
+      Context.insert<InstFakeUse>(T_Lo);
+      Context.insert<InstFakeUse>(T_Hi);
+    } else {
+      auto *T = makeReg(DestTy);
+      lowerLoad(InstLoad::create(Func, T,
+                                 formMemoryOperand(Instr->getArg(0), DestTy)));
+      _sync();
+      _mov(Dest, T);
+      // Adding a fake-use of T to ensure the atomic load is not removed if Dest
+      // is unused.
+      Context.insert<InstFakeUse>(T);
+    }
     return;
   }
-  case Intrinsics::AtomicFence:
-    UnimplementedLoweringError(this, Instr);
-    return;
-  case Intrinsics::AtomicFenceAll:
-    // NOTE: FenceAll should prevent and load/store from being moved across the
-    // fence (both atomic and non-atomic). The InstMIPS32Mfence instruction is
-    // currently marked coarsely as "HasSideEffects".
-    UnimplementedLoweringError(this, Instr);
+  case Intrinsics::AtomicStore: {
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal stores are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(2)))) {
+      Func->setError("Unexpected memory ordering for AtomicStore");
+      return;
+    }
+    auto *Val = Instr->getArg(0);
+    auto Ty = Val->getType();
+    if (Ty == IceType_i64) {
+      Variable *ValHi, *ValLo;
+      _sync();
+      if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(Val)) {
+        const uint64_t Value = C64->getValue();
+        uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
+        uint64_t Lower32Bits = Value & 0xFFFFFFFF;
+        ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
+        ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
+      } else {
+        auto *Val64 = llvm::cast<Variable64On32>(Val);
+        ValLo = legalizeToReg(loOperand(Val64));
+        ValHi = legalizeToReg(hiOperand(Val64));
+      }
+
+      auto *Base = legalizeToReg(Instr->getArg(1));
+      auto *AddrLo = OperandMIPS32Mem::create(
+          Func, IceType_i32, Base,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
+      auto *AddrHi = OperandMIPS32Mem::create(
+          Func, IceType_i32, Base,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
+      lowerStore(InstStore::create(Func, ValLo, AddrLo, IceType_i32));
+      lowerStore(InstStore::create(Func, ValHi, AddrHi, IceType_i32));
+      _sync();
+    } else {
+      _sync();
+      auto *Val = legalizeToReg(Instr->getArg(0));
+      lowerStore(InstStore::create(
+          Func, Val, formMemoryOperand(Instr->getArg(1), DestTy)));
+      _sync();
+    }
     return;
-  case Intrinsics::AtomicIsLockFree: {
-    UnimplementedLoweringError(this, Instr);
+  }
+  case Intrinsics::AtomicCmpxchg: {
+    assert(isScalarIntegerType(DestTy));
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)),
+            getConstantMemoryOrder(Instr->getArg(4)))) {
+      Func->setError("Unexpected memory ordering for AtomicCmpxchg");
+      return;
+    }
+
+    InstMIPS32Label *Exit = InstMIPS32Label::create(Func, this);
+    InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
+    constexpr CfgNode *NoTarget = nullptr;
+    auto *New = Instr->getArg(2);
+    auto *Expected = Instr->getArg(1);
+    auto *ActualAddress = Instr->getArg(0);
+
+    if (DestTy == IceType_i64) {
+      InstMIPS32Label *Retry1 = InstMIPS32Label::create(Func, this);
+      auto *T1 = I32Reg();
+      auto *T2 = I32Reg();
+      _sync();
+      Variable *ValHi, *ValLo, *ExpectedLo, *ExpectedHi;
+      if (llvm::isa<ConstantUndef>(Expected)) {
+        ExpectedLo = legalizeToReg(Ctx->getConstantZero(IceType_i32));
+        ExpectedHi = legalizeToReg(Ctx->getConstantZero(IceType_i32));
+      } else {
+        auto *Expected64 = llvm::cast<Variable64On32>(Expected);
+        ExpectedLo = legalizeToReg(loOperand(Expected64));
+        ExpectedHi = legalizeToReg(hiOperand(Expected64));
+      }
+      if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(New)) {
+        const uint64_t Value = C64->getValue();
+        uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
+        uint64_t Lower32Bits = Value & 0xFFFFFFFF;
+        ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
+        ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
+      } else {
+        auto *Val = llvm::cast<Variable64On32>(New);
+        ValLo = legalizeToReg(loOperand(Val));
+        ValHi = legalizeToReg(hiOperand(Val));
+      }
+      auto *Dest64 = llvm::cast<Variable64On32>(Dest);
+      auto *BaseR = legalizeToReg(ActualAddress);
+      auto *AddrLo = OperandMIPS32Mem::create(
+          Func, IceType_i32, BaseR,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
+      auto *AddrHi = OperandMIPS32Mem::create(
+          Func, IceType_i32, BaseR,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
+      Context.insert(Retry);
+      _ll(T1, AddrLo);
+      _br(NoTarget, NoTarget, T1, ExpectedLo, Exit, CondMIPS32::Cond::NE);
+      _sc(ValLo, AddrLo);
+      _br(NoTarget, NoTarget, ValLo, getZero(), Retry, CondMIPS32::Cond::EQ);
+      _mov(Dest64->getLo(), T1);
+      Context.insert(Retry1);
+      _ll(T2, AddrHi);
+      _br(NoTarget, NoTarget, T2, ExpectedHi, Exit, CondMIPS32::Cond::NE);
+      _sc(ValHi, AddrHi);
+      _br(NoTarget, NoTarget, ValHi, getZero(), Retry1, CondMIPS32::Cond::EQ);
+      _mov(Dest64->getHi(), T2);
+      Context.insert<InstFakeUse>(getZero());
+      Context.insert(Exit);
+      _sync();
+    } else if (DestTy == IceType_i8 || DestTy == IceType_i16) {
+      auto *NewR = legalizeToReg(New);
+      auto *ExpectedR = legalizeToReg(Expected);
+      auto *ActualAddressR = legalizeToReg(ActualAddress);
+      const uint32_t ShiftAmount =
+          (INT32_BITS - CHAR_BITS * typeWidthInBytes(DestTy));
+      const uint32_t Mask = (1 << (CHAR_BITS * typeWidthInBytes(DestTy))) - 1;
+      auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
+      auto *T1 = I32Reg();
+      auto *T2 = I32Reg();
+      auto *T3 = I32Reg();
+      auto *T4 = I32Reg();
+      auto *T5 = I32Reg();
+      auto *T6 = I32Reg();
+      auto *T7 = I32Reg();
+      auto *T8 = I32Reg();
+      auto *T9 = I32Reg();
+      _sync();
+      _addiu(RegAt, getZero(), -4);
+      _and(T1, ActualAddressR, RegAt);
+      _andi(RegAt, ActualAddressR, 3);
+      _sll(T2, RegAt, 3);
+      _ori(RegAt, getZero(), Mask);
+      _sllv(T3, RegAt, T2);
+      _nor(T4, getZero(), T3);
+      _andi(RegAt, ExpectedR, Mask);
+      _sllv(T5, RegAt, T2);
+      _andi(RegAt, NewR, Mask);
+      _sllv(T6, RegAt, T2);
+      Context.insert(Retry);
+      _ll(T7, formMemoryOperand(T1, DestTy));
+      _and(T8, T7, T3);
+      _br(NoTarget, NoTarget, T8, T5, Exit, CondMIPS32::Cond::NE);
+      _and(RegAt, T7, T4);
+      _or(T9, RegAt, T6);
+      _sc(T9, formMemoryOperand(T1, DestTy));
+      _br(NoTarget, NoTarget, getZero(), T9, Retry, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      Context.insert(Exit);
+      _srlv(RegAt, T8, T2);
+      _sll(RegAt, RegAt, ShiftAmount);
+      _sra(RegAt, RegAt, ShiftAmount);
+      _mov(Dest, RegAt);
+      _sync();
+      Context.insert<InstFakeUse>(ExpectedR);
+      Context.insert<InstFakeUse>(NewR);
+    } else {
+      auto *T1 = I32Reg();
+      auto *NewR = legalizeToReg(New);
+      auto *ExpectedR = legalizeToReg(Expected);
+      auto *ActualAddressR = legalizeToReg(ActualAddress);
+      _sync();
+      Context.insert(Retry);
+      _ll(T1, formMemoryOperand(ActualAddressR, DestTy));
+      _br(NoTarget, NoTarget, T1, ExpectedR, Exit, CondMIPS32::Cond::NE);
+      _sc(NewR, formMemoryOperand(ActualAddressR, DestTy));
+      _br(NoTarget, NoTarget, NewR, getZero(), Retry, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      Context.insert(Exit);
+      _mov(Dest, T1);
+      _sync();
+      Context.insert<InstFakeUse>(ExpectedR);
+      Context.insert<InstFakeUse>(NewR);
+    }
     return;
   }
-  case Intrinsics::AtomicLoad: {
-    UnimplementedLoweringError(this, Instr);
+  case Intrinsics::AtomicRMW: {
+    assert(isScalarIntegerType(DestTy));
+    // We require the memory address to be naturally aligned. Given that is the
+    // case, then normal loads are atomic.
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)))) {
+      Func->setError("Unexpected memory ordering for AtomicRMW");
+      return;
+    }
+
+    constexpr CfgNode *NoTarget = nullptr;
+    InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
+    auto Operation = static_cast<Intrinsics::AtomicRMWOperation>(
+        llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue());
+    auto *New = Instr->getArg(2);
+    auto *ActualAddress = Instr->getArg(1);
+
+    if (DestTy == IceType_i64) {
+      InstMIPS32Label *Retry1 = InstMIPS32Label::create(Func, this);
+      _sync();
+      Variable *ValHi, *ValLo;
+      if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(New)) {
+        const uint64_t Value = C64->getValue();
+        uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
+        uint64_t Lower32Bits = Value & 0xFFFFFFFF;
+        ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
+        ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
+      } else {
+        auto *Val = llvm::cast<Variable64On32>(New);
+        ValLo = legalizeToReg(loOperand(Val));
+        ValHi = legalizeToReg(hiOperand(Val));
+      }
+      auto *Dest64 = llvm::cast<Variable64On32>(Dest);
+      auto *BaseR = legalizeToReg(ActualAddress);
+      auto *AddrLo = OperandMIPS32Mem::create(
+          Func, IceType_i32, BaseR,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
+      auto *AddrHi = OperandMIPS32Mem::create(
+          Func, IceType_i32, BaseR,
+          llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
+      auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
+      auto *T1 = I32Reg();
+      auto *T2 = I32Reg();
+      auto *T3 = I32Reg();
+      Context.insert(Retry);
+      _ll(T1, AddrLo);
+      if (Operation == Intrinsics::AtomicExchange) {
+        _mov(RegAt, ValLo);
+      } else if (Operation == Intrinsics::AtomicAdd) {
+        createArithInst(Operation, RegAt, T1, ValLo);
+        _sltu(T2, RegAt, T1);
+      } else if (Operation == Intrinsics::AtomicSub) {
+        createArithInst(Operation, RegAt, T1, ValLo);
+        _sltu(T2, T1, ValLo);
+      } else {
+        createArithInst(Operation, RegAt, T1, ValLo);
+      }
+      _sc(RegAt, AddrLo);
+      _br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      _mov(Dest64->getLo(), T1);
+      Context.insert(Retry1);
+      _ll(T3, AddrHi);
+      if (Operation == Intrinsics::AtomicAdd ||
+          Operation == Intrinsics::AtomicSub) {
+        _addu(RegAt, T2, ValHi);
+        createArithInst(Operation, RegAt, T3, RegAt);
+      } else if (Operation == Intrinsics::AtomicExchange) {
+        _mov(RegAt, ValHi);
+      } else {
+        createArithInst(Operation, RegAt, T3, ValHi);
+      }
+      _sc(RegAt, AddrHi);
+      _br(NoTarget, NoTarget, RegAt, getZero(), Retry1, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      _mov(Dest64->getHi(), T3);
+      Context.insert<InstFakeUse>(ValLo);
+      Context.insert<InstFakeUse>(ValHi);
+      _sync();
+    } else if (DestTy == IceType_i8 || DestTy == IceType_i16) {
+      const uint32_t ShiftAmount =
+          INT32_BITS - (CHAR_BITS * typeWidthInBytes(DestTy));
+      const uint32_t Mask = (1 << (CHAR_BITS * typeWidthInBytes(DestTy))) - 1;
+      auto *NewR = legalizeToReg(New);
+      auto *ActualAddressR = legalizeToReg(ActualAddress);
+      auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
+      auto *T1 = I32Reg();
+      auto *T2 = I32Reg();
+      auto *T3 = I32Reg();
+      auto *T4 = I32Reg();
+      auto *T5 = I32Reg();
+      auto *T6 = I32Reg();
+      auto *T7 = I32Reg();
+      _sync();
+      _addiu(RegAt, getZero(), -4);
+      _and(T1, ActualAddressR, RegAt);
+      _andi(RegAt, ActualAddressR, 3);
+      _sll(T2, RegAt, 3);
+      _ori(RegAt, getZero(), Mask);
+      _sllv(T3, RegAt, T2);
+      _nor(T4, getZero(), T3);
+      _sllv(T5, NewR, T2);
+      Context.insert(Retry);
+      _ll(T6, formMemoryOperand(T1, DestTy));
+      if (Operation != Intrinsics::AtomicExchange) {
+        createArithInst(Operation, RegAt, T6, T5);
+        _and(RegAt, RegAt, T3);
+      }
+      _and(T7, T6, T4);
+      if (Operation == Intrinsics::AtomicExchange) {
+        _or(RegAt, T7, T5);
+      } else {
+        _or(RegAt, T7, RegAt);
+      }
+      _sc(RegAt, formMemoryOperand(T1, DestTy));
+      _br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      _and(RegAt, T6, T3);
+      _srlv(RegAt, RegAt, T2);
+      _sll(RegAt, RegAt, ShiftAmount);
+      _sra(RegAt, RegAt, ShiftAmount);
+      _mov(Dest, RegAt);
+      _sync();
+      Context.insert<InstFakeUse>(NewR);
+      Context.insert<InstFakeUse>(Dest);
+    } else {
+      auto *T1 = I32Reg();
+      auto *T2 = I32Reg();
+      auto *NewR = legalizeToReg(New);
+      auto *ActualAddressR = legalizeToReg(ActualAddress);
+      _sync();
+      Context.insert(Retry);
+      _ll(T1, formMemoryOperand(ActualAddressR, DestTy));
+      if (Operation == Intrinsics::AtomicExchange) {
+        _mov(T2, NewR);
+      } else {
+        createArithInst(Operation, T2, T1, NewR);
+      }
+      _sc(T2, formMemoryOperand(ActualAddressR, DestTy));
+      _br(NoTarget, NoTarget, T2, getZero(), Retry, CondMIPS32::Cond::EQ);
+      Context.insert<InstFakeUse>(getZero());
+      _mov(Dest, T1);
+      _sync();
+      Context.insert<InstFakeUse>(NewR);
+      Context.insert<InstFakeUse>(Dest);
+    }
     return;
   }
-  case Intrinsics::AtomicRMW:
-    UnimplementedLoweringError(this, Instr);
+  case Intrinsics::AtomicFence:
+  case Intrinsics::AtomicFenceAll:
+    assert(Dest == nullptr);
+    _sync();
     return;
-  case Intrinsics::AtomicStore: {
-    UnimplementedLoweringError(this, Instr);
+  case Intrinsics::AtomicIsLockFree: {
+    Operand *ByteSize = Instr->getArg(0);
+    auto *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize);
+    auto *T = I32Reg();
+    if (CI == nullptr) {
+      // The PNaCl ABI requires the byte size to be a compile-time constant.
+      Func->setError("AtomicIsLockFree byte size should be compile-time const");
+      return;
+    }
+    static constexpr int32_t NotLockFree = 0;
+    static constexpr int32_t LockFree = 1;
+    int32_t Result = NotLockFree;
+    switch (CI->getValue()) {
+    case 1:
+    case 2:
+    case 4:
+    case 8:
+      Result = LockFree;
+      break;
+    }
+    _addiu(T, getZero(), Result);
+    _mov(Dest, T);
     return;
   }
   case Intrinsics::Bswap: {
@@ -4935,7 +5347,7 @@ void TargetMIPS32::lowerSelect(const InstSelect *Instr) {
     _mov(Dest, SrcFR);
     break;
   default:
-    UnimplementedLoweringError(this, Instr);
+    llvm::report_fatal_error("Select: Invalid type.");
   }
 }
 
@@ -5297,7 +5709,7 @@ Operand *TargetMIPS32::legalize(Operand *From, LegalMask Allowed,
       // using a lui-ori instructions.
       Variable *Reg = makeReg(Ty, RegNum);
       if (isInt<16>(int32_t(Value))) {
-        Variable *Zero = getPhysicalRegister(RegMIPS32::Reg_ZERO, Ty);
+        Variable *Zero = makeReg(Ty, RegMIPS32::Reg_ZERO);
         Context.insert<InstFakeDef>(Zero);
         _addiu(Reg, Zero, Value);
       } else {

src/IceTargetLoweringMIPS32.h

@@ -175,6 +175,10 @@ public:
 
   void _br(CfgNode *Target) { Context.insert<InstMIPS32Br>(Target); }
 
+  void _br(CfgNode *Target, const InstMIPS32Label *Label) {
+    Context.insert<InstMIPS32Br>(Target, Label);
+  }
+
   void _br(CfgNode *TargetTrue, CfgNode *TargetFalse, Operand *Src0,
            Operand *Src1, CondMIPS32::Cond Condition) {
     Context.insert<InstMIPS32Br>(TargetTrue, TargetFalse, Src0, Src1,
@@ -329,6 +333,10 @@ public:
     Context.insert<InstMIPS32Ldc1>(Value, Mem, Reloc);
   }
 
+  void _ll(Variable *Value, OperandMIPS32Mem *Mem) {
+    Context.insert<InstMIPS32Ll>(Value, Mem);
+  }
+
   void _lw(Variable *Value, OperandMIPS32Mem *Mem) {
     Context.insert<InstMIPS32Lw>(Value, Mem);
   }
@@ -474,6 +482,10 @@ public:
     Context.insert<InstMIPS32Ori>(Dest, Src, Imm);
   }
 
+  InstMIPS32Sc *_sc(Variable *Value, OperandMIPS32Mem *Mem) {
+    return Context.insert<InstMIPS32Sc>(Value, Mem);
+  }
+
   void _sdc1(Variable *Value, OperandMIPS32Mem *Mem) {
     Context.insert<InstMIPS32Sdc1>(Value, Mem);
   }
@@ -550,6 +562,8 @@ public:
     Context.insert<InstMIPS32Swc1>(Value, Mem);
   }
 
+  void _sync() { Context.insert<InstMIPS32Sync>(); }
+
   void _teq(Variable *Src0, Variable *Src1, uint32_t TrapCode) {
     Context.insert<InstMIPS32Teq>(Src0, Src1, TrapCode);
   }
@@ -746,6 +760,8 @@ protected:
   void lowerFcmp(const InstFcmp *Instr) override;
   void lowerIcmp(const InstIcmp *Instr) override;
   void lower64Icmp(const InstIcmp *Instr);
+  void createArithInst(Intrinsics::AtomicRMWOperation Operation, Variable *Dest,
+                       Variable *Src0, Variable *Src1);
   void lowerIntrinsicCall(const InstIntrinsicCall *Instr) override;
   void lowerInsertElement(const InstInsertElement *Instr) override;
   void lowerLoad(const InstLoad *Instr) override;

tests_lit/llvm2ice_tests/nacl-atomic-intrinsics.ll

@@ -26,6 +26,18 @@
 ; RUN:   | %if --need=allow_dump --need=target_ARM32 --command FileCheck %s \
 ; RUN:   --check-prefix=ARM32
 
+; RUN: %if --need=allow_dump --need=target_MIPS32 --command %p2i --filetype=asm\
+; RUN:   --target mips32 -i %s --args -O2 \
+; RUN:   -allow-externally-defined-symbols \
+; RUN:   | %if --need=allow_dump --need=target_MIPS32 --command FileCheck %s \
+; RUN:   --check-prefix=MIPS32O2 --check-prefix=MIPS32
+
+; RUN: %if --need=allow_dump --need=target_MIPS32 --command %p2i --filetype=asm\
+; RUN:   --target mips32 -i %s --args -Om1 \
+; RUN:   -allow-externally-defined-symbols \
+; RUN:   | %if --need=allow_dump --need=target_MIPS32 --command FileCheck %s \
+; RUN:   --check-prefix=MIPS32OM1 --check-prefix=MIPS32
+
 declare i8 @llvm.nacl.atomic.load.i8(i8*, i32)
 declare i16 @llvm.nacl.atomic.load.i16(i16*, i32)
 declare i32 @llvm.nacl.atomic.load.i32(i32*, i32)
@@ -76,6 +88,9 @@ entry:
 ; ARM32-LABEL: test_atomic_load_8
 ; ARM32: ldrb r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_8
+; MIPS32: lb
+; MIPS32: sync
 
 define internal i32 @test_atomic_load_16(i32 %iptr) {
 entry:
@@ -91,6 +106,9 @@ entry:
 ; ARM32-LABEL: test_atomic_load_16
 ; ARM32: ldrh r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_16
+; MIPS32: lh
+; MIPS32: sync
 
 define internal i32 @test_atomic_load_32(i32 %iptr) {
 entry:
@@ -104,6 +122,9 @@ entry:
 ; ARM32-LABEL: test_atomic_load_32
 ; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_32
+; MIPS32: lw
+; MIPS32: sync
 
 define internal i64 @test_atomic_load_64(i32 %iptr) {
 entry:
@@ -117,6 +138,10 @@ entry:
 ; ARM32-LABEL: test_atomic_load_64
 ; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_64
+; MIPS32: lw
+; MIPS32: lw
+; MIPS32: sync
 
 define internal i32 @test_atomic_load_32_with_arith(i32 %iptr) {
 entry:
@@ -139,6 +164,9 @@ next:
 ; ARM32-LABEL: test_atomic_load_32_with_arith
 ; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_32_with_arith
+; MIPS32: lw
+; MIPS32: sync
 
 define internal i32 @test_atomic_load_32_ignored(i32 %iptr) {
 entry:
@@ -155,6 +183,9 @@ entry:
 ; ARM32-LABEL: test_atomic_load_32_ignored
 ; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_32_ignored
+; MIPS32: lw
+; MIPS32: sync
 
 define internal i64 @test_atomic_load_64_ignored(i32 %iptr) {
 entry:
@@ -168,6 +199,10 @@ entry:
 ; ARM32-LABEL: test_atomic_load_64_ignored
 ; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_load_64_ignored
+; MIPS32: lw
+; MIPS32: lw
+; MIPS32: sync
 
 ;;; Store
 
@@ -185,6 +220,10 @@ entry:
 ; ARM32: dmb
 ; ARM32: strb r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_store_8
+; MIPS32: sync
+; MIPS32: sb
+; MIPS32: sync
 
 define internal void @test_atomic_store_16(i32 %iptr, i32 %v) {
 entry:
@@ -200,6 +239,10 @@ entry:
 ; ARM32: dmb
 ; ARM32: strh r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_store_16
+; MIPS32: sync
+; MIPS32: sh
+; MIPS32: sync
 
 define internal void @test_atomic_store_32(i32 %iptr, i32 %v) {
 entry:
@@ -214,6 +257,10 @@ entry:
 ; ARM32: dmb
 ; ARM32: str r{{[0-9]+}}, [r{{[0-9]+}}
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_store_32
+; MIPS32: sync
+; MIPS32: sw
+; MIPS32: sync
 
 define internal void @test_atomic_store_64(i32 %iptr, i64 %v) {
 entry:
@@ -232,6 +279,11 @@ entry:
 ; ARM32: cmp [[S]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_store_64
+; MIPS32: sync
+; MIPS32: sw
+; MIPS32: sw
+; MIPS32: sync
 
 define internal void @test_atomic_store_64_const(i32 %iptr) {
 entry:
@@ -256,6 +308,14 @@ entry:
 ; ARM32: cmp [[S]], #0
 ; ARM32: bne .L[[RETRY]]
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_store_64_const
+; MIPS32: sync
+; MIPS32: lui	{{.*}}, 29646
+; MIPS32: ori	{{.*}},{{.*}}, 12274
+; MIPS32: addiu	{{.*}}, $zero, 2874
+; MIPS32: sw
+; MIPS32: sw
+; MIPS32: sync
 
 ;;; RMW
 
@@ -280,6 +340,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_add_16(i32 %iptr, i32 %v) {
 entry:
@@ -299,6 +381,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_add_32(i32 %iptr, i32 %v) {
 entry:
@@ -316,6 +420,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_add_64(i32 %iptr, i64 %v) {
 entry:
@@ -346,6 +457,19 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sltu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define internal i64 @test_atomic_rmw_add_64_global(i64 %v) {
@@ -363,6 +487,19 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_64_global
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sltu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ; Test with some more register pressure. When we have an alloca, ebp is
 ; used to manage the stack frame, so it cannot be used as a register either.
@@ -404,6 +541,19 @@ eblock:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_64_alloca
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sltu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_add_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -422,6 +572,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_add_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ; Atomic RMW 64 needs to be expanded into its own loop.
 ; Make sure that works w/ non-trivial function bodies.
@@ -462,6 +619,19 @@ err:
 ; ARM32: bne
 ; ARM32: dmb
 ; ARM32: b
+; MIPS32-LABEL: test_atomic_rmw_add_64_loop
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: sltu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: addu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;; sub
 
@@ -484,6 +654,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_sub_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: subu
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_sub_16(i32 %iptr, i32 %v) {
 entry:
@@ -504,6 +696,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_sub_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: subu
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_sub_32(i32 %iptr, i32 %v) {
 entry:
@@ -522,6 +736,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_sub_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: subu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_sub_64(i32 %iptr, i64 %v) {
 entry:
@@ -547,6 +768,19 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_sub_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: subu
+; MIPS32: sltu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: addu
+; MIPS32: subu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_sub_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -565,6 +799,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_sub_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: subu
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;; or
 
@@ -590,6 +831,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define internal i32 @test_atomic_rmw_or_8_global(i32 %v) {
@@ -610,6 +873,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_8_global
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_or_16(i32 %iptr, i32 %v) {
 entry:
@@ -631,6 +916,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define internal i32 @test_atomic_rmw_or_16_global(i32 %v) {
@@ -651,6 +958,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_16_global
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_or_32(i32 %iptr, i32 %v) {
 entry:
@@ -670,6 +999,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ; Same test as above, but with a global address to test FakeUse issues.
 define internal i32 @test_atomic_rmw_or_32_global(i32 %v) {
@@ -688,6 +1024,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_32_global
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_or_64(i32 %iptr, i64 %v) {
 entry:
@@ -713,6 +1056,17 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_or_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -735,6 +1089,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_or_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;; and
 
@@ -758,6 +1119,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_and_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_and_16(i32 %iptr, i32 %v) {
 entry:
@@ -779,6 +1162,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_and_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_and_32(i32 %iptr, i32 %v) {
 entry:
@@ -798,6 +1203,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_and_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_and_64(i32 %iptr, i64 %v) {
 entry:
@@ -823,6 +1235,17 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_and_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_and_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -843,6 +1266,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_and_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;; xor
 
@@ -866,6 +1296,28 @@ entry:
 ; ARM32: strexb
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xor_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xor_16(i32 %iptr, i32 %v) {
 entry:
@@ -887,6 +1339,28 @@ entry:
 ; ARM32: strexh
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xor_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: and
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xor_32(i32 %iptr, i32 %v) {
 entry:
@@ -906,6 +1380,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xor_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_xor_64(i32 %iptr, i64 %v) {
 entry:
@@ -931,6 +1412,17 @@ entry:
 ; ARM32: strexd r{{[0-9]+}}, r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}]
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xor_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xor_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -950,6 +1442,13 @@ entry:
 ; ARM32: strex
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xor_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: xor
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;; exchange
 
@@ -970,6 +1469,26 @@ entry:
 ; ARM32: cmp
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xchg_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xchg_16(i32 %iptr, i32 %v) {
 entry:
@@ -988,6 +1507,26 @@ entry:
 ; ARM32: cmp
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xchg_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: and
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xchg_32(i32 %iptr, i32 %v) {
 entry:
@@ -1004,6 +1543,13 @@ entry:
 ; ARM32: cmp
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xchg_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: move
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_rmw_xchg_64(i32 %iptr, i64 %v) {
 entry:
@@ -1026,6 +1572,17 @@ entry:
 ; ARM32: cmp
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xchg_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: move
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: ll
+; MIPS32: move
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i32 @test_atomic_rmw_xchg_32_ignored(i32 %iptr, i32 %v) {
 entry:
@@ -1044,6 +1601,13 @@ entry:
 ; ARM32: cmp
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_rmw_xchg_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: move
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 ;;;; Cmpxchg
 
@@ -1073,6 +1637,30 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_8
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, $zero, 255
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: andi	{{.*}}, {{.*}}, 255
+; MIPS32: sllv
+; MIPS32: andi	{{.*}}, {{.*}}, 255
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: bne
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	$zero, {{.*}}, {{.*}}
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 24
+; MIPS32: sra	{{.*}}, {{.*}}, 24
+; MIPS32: sync
 
 define internal i32 @test_atomic_cmpxchg_16(i32 %iptr, i32 %expected,
                                             i32 %desired) {
@@ -1098,6 +1686,30 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_16
+; MIPS32: sync
+; MIPS32: addiu	{{.*}}, $zero, -4
+; MIPS32: and
+; MIPS32: andi	{{.*}}, {{.*}}, 3
+; MIPS32: sll	{{.*}}, {{.*}}, 3
+; MIPS32: ori	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: nor
+; MIPS32: andi	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: andi	{{.*}}, {{.*}}, 65535
+; MIPS32: sllv
+; MIPS32: ll
+; MIPS32: and
+; MIPS32: bne
+; MIPS32: and
+; MIPS32: or
+; MIPS32: sc
+; MIPS32: beq	$zero, {{.*}}, {{.*}}
+; MIPS32: srlv
+; MIPS32: sll	{{.*}}, {{.*}}, 16
+; MIPS32: sra	{{.*}}, {{.*}}, 16
+; MIPS32: sync
 
 define internal i32 @test_atomic_cmpxchg_32(i32 %iptr, i32 %expected,
                                             i32 %desired) {
@@ -1119,6 +1731,13 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_32
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_cmpxchg_64(i32 %iptr, i64 %expected,
                                             i64 %desired) {
@@ -1148,6 +1767,18 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_64
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: sync
+
 
 define internal i64 @test_atomic_cmpxchg_64_undef(i32 %iptr, i64 %desired) {
 entry:
@@ -1170,6 +1801,17 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_64_undef
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: sync
 
 ; Test a case where %old really does need to be copied out of edx:eax.
 define internal void @test_atomic_cmpxchg_64_store(
@@ -1203,6 +1845,18 @@ entry:
 ; ARM32: dmb
 ; ARM32: str
 ; ARM32: str
+; MIPS32-LABEL: test_atomic_cmpxchg_64_store
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: sync
+
 
 ; Test with some more register pressure. When we have an alloca, ebp is
 ; used to manage the stack frame, so it cannot be used as a register either.
@@ -1246,6 +1900,17 @@ eblock:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_64_alloca
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: sync
 
 define internal i32 @test_atomic_cmpxchg_32_ignored(i32 %iptr, i32 %expected,
                                                     i32 %desired) {
@@ -1267,6 +1932,13 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_32_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq	{{.*}}, $zero, {{.*}}
+; MIPS32: sync
 
 define internal i64 @test_atomic_cmpxchg_64_ignored(i32 %iptr, i64 %expected,
                                                     i64 %desired) {
@@ -1293,6 +1965,17 @@ entry:
 ; ARM32: cmp [[SUCCESS]], #0
 ; ARM32: bne
 ; ARM32: dmb
+; MIPS32-LABEL: test_atomic_cmpxchg_64_ignored
+; MIPS32: sync
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: ll
+; MIPS32: bne
+; MIPS32: sc
+; MIPS32: beq
+; MIPS32: sync
 
 ;;;; Fence and is-lock-free.
 
@@ -1305,6 +1988,8 @@ entry:
 ; CHECK: mfence
 ; ARM32-LABEL: test_atomic_fence
 ; ARM32: dmb sy
+; MIPS32-LABEL: test_atomic_fence
+; MIPS32: sync
 
 define internal void @test_atomic_fence_all() {
 entry:
@@ -1315,6 +2000,8 @@ entry:
 ; CHECK: mfence
 ; ARM32-LABEL: test_atomic_fence_all
 ; ARM32: dmb sy
+; MIPS32-LABEL: test_atomic_fence_all
+; MIPS32: sync
 
 define internal i32 @test_atomic_is_lock_free(i32 %iptr) {
 entry:
@@ -1327,6 +2014,8 @@ entry:
 ; CHECK: mov {{.*}},0x1
 ; ARM32-LABEL: test_atomic_is_lock_free
 ; ARM32: mov {{.*}}, #1
+; MIPS32-LABEL: test_atomic_is_lock_free
+; MIPS32: addiu {{.*}}, $zero, 1
 
 define internal i32 @test_not_lock_free(i32 %iptr) {
 entry:
@@ -1339,6 +2028,8 @@ entry:
 ; CHECK: mov {{.*}},0x0
 ; ARM32-LABEL: test_not_lock_free
 ; ARM32: mov {{.*}}, #0
+; MIPS32-LABEL: test_not_lock_free
+; MIPS32: addiu {{.*}}, $zero, 0
 
 define internal i32 @test_atomic_is_lock_free_ignored(i32 %iptr) {
 entry:
@@ -1355,6 +2046,9 @@ entry:
 ; ARM32O2-LABEL: test_atomic_is_lock_free_ignored
 ; ARM32O2-NOT: mov {{.*}}, #1
 ; ARM32O2: mov {{.*}}, #0
+; MIPS32O2-LABEL: test_atomic_is_lock_free
+; MIPS32O2-NOT: addiu {{.*}}, $zero, 1
+; MIPS32O2: addiu {{.*}}, $zero, 0
 
 ; TODO(jvoung): at some point we can take advantage of the
 ; fact that nacl.atomic.is.lock.free will resolve to a constant