Skip to content

S
swiftshader
Commit cfa628b5 by Andrew Scull
Options

Inline memove for small constant sizes and refactor memcpy and memset.

The memory intrinsics are only optimized at -O1 and higher unless the -fmem-intrin-opt flag is set to force to optimization to take place. This change also introduces the xchg instruction for two register operands. This is no longer used in the memory intrinsic lowering (or by anything else) but the implementation is left for future use. BUG= R=jvoung@chromium.org, stichnot@chromium.org Review URL: https://codereview.chromium.org/1278173009.
parent 209318af
Showing with 883 additions and 199 deletions
Makefile.standalone
...@@ -227,6 +227,9 @@ UNITTEST_SRCS = \ ...@@ -227,6 +227,9 @@ UNITTEST_SRCS = \
# The X86 assembler tests take too long to compile. Given how infrequently the # The X86 assembler tests take too long to compile. Given how infrequently the
# assembler will change, we disable them. # assembler will change, we disable them.
ifdef CHECK_X86_ASM ifdef CHECK_X86_ASM
ifndef DEBUG
$(error Run check-unit with DEBUG=1 lest your machine perish)
endif
UNITTEST_SRCS += AssemblerX8632/LowLevel.cpp \ UNITTEST_SRCS += AssemblerX8632/LowLevel.cpp \
AssemblerX8632/DataMov.cpp \ AssemblerX8632/DataMov.cpp \
AssemblerX8632/Locked.cpp \ AssemblerX8632/Locked.cpp \
...@@ -350,11 +353,14 @@ check-xtest: $(OBJDIR)/pnacl-sz make_symlink runtime ...@@ -350,11 +353,14 @@ check-xtest: $(OBJDIR)/pnacl-sz make_symlink runtime
# Do all native/sse2 tests, but only test_vector_ops for native/sse4.1. # Do all native/sse2 tests, but only test_vector_ops for native/sse4.1.
# For (slow) sandboxed tests, limit to Om1/sse4.1. # For (slow) sandboxed tests, limit to Om1/sse4.1.
# TODO(jpp): implement x8664 sandbox, then enable xtests. # TODO(jpp): implement x8664 sandbox, then enable xtests.
# TODO(jpp): reenable the x86-64 tests.
./pydir/crosstest_generator.py -v --lit \ ./pydir/crosstest_generator.py -v --lit \
--toolchain-root $(TOOLCHAIN_ROOT) \ --toolchain-root $(TOOLCHAIN_ROOT) \
-i x8632,native,sse2 -i x8632,native,sse4.1,test_vector_ops \ -i x8632,native,sse2 \
-i x8632,native,sse4.1,test_vector_ops \
-i x8632,sandbox,sse4.1,Om1 \ -i x8632,sandbox,sse4.1,Om1 \
-i x8664,native,sse2 -i x8664,native,sse4.1,test_vector_ops \ -e x8664,native,sse2 \
-e x8664,native,sse4.1,test_vector_ops \
-e x8664,native,sse2,test_global \ -e x8664,native,sse2,test_global \
-i arm32,native,neon,simple_loop \ -i arm32,native,neon,simple_loop \
-i arm32,native,neon,mem_intrin \ -i arm32,native,neon,mem_intrin \
......
crosstest/crosstest.cfg
...@@ -5,6 +5,7 @@ test: simple_loop.c ...@@ -5,6 +5,7 @@ test: simple_loop.c
[mem_intrin] [mem_intrin]
driver: mem_intrin_main.cpp driver: mem_intrin_main.cpp
test: mem_intrin.cpp test: mem_intrin.cpp
flags: --sz=-fmem-intrin-opt
[test_arith] [test_arith]
driver: test_arith_main.cpp driver: test_arith_main.cpp
......
crosstest/mem_intrin.cpp
...@@ -40,30 +40,6 @@ fletcher_checksum(uint8_t *buf, SizeT length) { ...@@ -40,30 +40,6 @@ fletcher_checksum(uint8_t *buf, SizeT length) {
return (sum_of_sums << 8) | sum; return (sum_of_sums << 8) | sum;
} }
#define NWORDS 32
#define BYTE_LENGTH (NWORDS * sizeof(elem_t))
int memcpy_test_fixed_len(uint8_t init) {
elem_t buf[NWORDS];
elem_t buf2[NWORDS];
reset_buf((uint8_t *)buf, init, BYTE_LENGTH);
memcpy((void *)buf2, (void *)buf, BYTE_LENGTH);
return fletcher_checksum((uint8_t *)buf2, BYTE_LENGTH);
}
int memmove_test_fixed_len(uint8_t init) {
elem_t buf[NWORDS];
reset_buf((uint8_t *)buf, init, BYTE_LENGTH);
memmove((void *)(buf + 4), (void *)buf, BYTE_LENGTH - (4 * sizeof(elem_t)));
return fletcher_checksum((uint8_t *)buf + 4, BYTE_LENGTH - 4);
}
int memset_test_fixed_len(uint8_t init) {
elem_t buf[NWORDS];
memset((void *)buf, init, BYTE_LENGTH);
return fletcher_checksum((uint8_t *)buf, BYTE_LENGTH);
}
int memcpy_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length) { int memcpy_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length) {
reset_buf(buf, init, length); reset_buf(buf, init, length);
memcpy((void *)buf2, (void *)buf, length); memcpy((void *)buf2, (void *)buf, length);
...@@ -94,3 +70,33 @@ int memset_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length) { ...@@ -94,3 +70,33 @@ int memset_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length) {
memset((void *)buf2, init + 4, length); memset((void *)buf2, init + 4, length);
return fletcher_checksum(buf, length) + fletcher_checksum(buf2, length); return fletcher_checksum(buf, length) + fletcher_checksum(buf2, length);
} }
#define X(NBYTES) \
int memcpy_test_fixed_len_##NBYTES(uint8_t init) { \
uint8_t buf[NBYTES]; \
uint8_t buf2[NBYTES]; \
reset_buf(buf, init, NBYTES); \
memcpy((void *)buf2, (void *)buf, NBYTES); \
return fletcher_checksum(buf2, NBYTES); \
} \
\
int memmove_test_fixed_len_##NBYTES(uint8_t init) { \
uint8_t buf[NBYTES + 16]; \
uint8_t buf2[NBYTES + 16]; \
reset_buf(buf, init, NBYTES + 16); \
reset_buf(buf2, init, NBYTES + 16); \
/* Move up */ \
memmove((void *)(buf + 16), (void *)buf, NBYTES); \
/* Move down */ \
memmove((void *)buf2, (void *)(buf2 + 16), NBYTES); \
return fletcher_checksum(buf, NBYTES + 16) + \
fletcher_checksum(buf2, NBYTES + 16); \
} \
\
int memset_test_fixed_len_##NBYTES(uint8_t init) { \
uint8_t buf[NBYTES]; \
memset((void *)buf, init, NBYTES); \
return fletcher_checksum(buf, NBYTES); \
}
MEMINTRIN_SIZE_TABLE
#undef X
crosstest/mem_intrin.def 0 → 100644
#define MEMINTRIN_SIZE_TABLE \
X(0) \
X(1) \
X(2) \
X(3) \
X(4) \
X(5) \
X(6) \
X(7) \
X(8) \
X(9) \
X(10) \
X(11) \
X(12) \
X(13) \
X(14) \
X(15) \
X(16) \
X(17) \
X(18) \
X(19) \
X(20) \
X(21) \
X(22) \
X(23) \
X(24) \
X(25) \
X(26) \
X(27) \
X(28) \
X(29) \
X(30) \
X(31) \
X(32) \
X(33) \
X(34) \
X(35) \
X(36) \
X(37) \
X(38) \
X(39) \
X(40) \
X(41) \
X(42) \
X(43) \
X(44) \
X(45) \
X(46) \
X(47) \
X(48) \
X(49) \
X(50) \
X(51) \
X(52) \
X(53) \
X(54) \
X(55) \
X(56) \
X(57) \
X(58) \
X(59) \
X(60) \
X(61) \
X(62) \
X(63) \
X(64) \
X(65) \
X(66) \
X(67) \
X(68) \
X(69) \
X(70) \
X(71) \
X(72) \
X(73) \
X(74) \
X(75) \
X(76) \
X(77) \
X(78) \
X(79) \
X(80) \
X(81) \
X(82) \
X(83) \
X(84) \
X(85) \
X(86) \
X(87) \
X(88) \
X(89) \
X(90) \
X(91) \
X(92) \
X(93) \
X(94) \
X(95) \
X(96) \
X(97) \
X(98) \
X(99) \
X(100) \
X(101) \
X(102) \
X(103) \
X(104) \
X(105) \
X(106) \
X(107) \
X(108) \
X(109) \
X(110) \
X(111) \
X(112) \
X(113) \
X(114) \
X(115) \
X(116) \
X(117) \
X(118) \
X(119) \
X(120) \
X(121) \
X(122) \
X(123) \
X(124) \
X(125) \
X(126) \
X(127) \
X(128) \
X(129) \
X(130) \
X(131) \
X(132) \
X(133) \
X(134) \
X(135) \
X(136) \
X(137) \
X(138) \
X(139) \
X(140) \
X(141) \
X(142) \
X(143) \
X(144) \
X(145) \
X(146) \
X(147) \
X(148) \
X(149) \
X(150) \
X(151) \
X(152) \
X(153) \
X(154) \
X(155) \
X(156) \
X(157) \
X(158) \
X(159) \
X(160) \
X(161) \
X(162) \
X(163) \
X(164) \
X(165) \
X(166) \
X(167) \
X(168) \
X(169) \
X(170) \
X(171) \
X(172) \
X(173) \
X(174) \
X(175) \
X(176) \
X(177) \
X(178) \
X(179) \
X(180) \
X(181) \
X(182) \
X(183) \
X(184) \
X(185) \
X(186) \
X(187) \
X(188) \
X(189) \
X(190) \
X(191) \
X(192) \
X(193) \
X(194) \
X(195) \
X(196) \
X(197) \
X(198) \
X(199) \
X(200) \
X(201) \
X(202) \
X(203) \
X(204) \
X(205) \
X(206) \
X(207) \
X(208) \
X(209) \
X(210) \
X(211) \
X(212) \
X(213) \
X(214) \
X(215) \
X(216) \
X(217) \
X(218) \
X(219) \
X(220) \
X(221) \
X(222) \
X(223) \
X(224) \
X(225) \
X(226) \
X(227) \
X(228) \
X(229) \
X(230) \
X(231) \
X(232) \
X(233) \
X(234) \
X(235) \
X(236) \
X(237) \
X(238) \
X(239) \
X(240) \
X(241) \
X(242) \
X(243) \
X(244) \
X(245) \
X(246) \
X(247) \
X(248) \
X(249) \
X(250) \
X(251) \
X(252) \
X(253) \
X(254) \
X(255) \
X(256)
crosstest/mem_intrin.h
...@@ -6,10 +6,15 @@ ...@@ -6,10 +6,15 @@
*/ */
#include "xdefs.h" #include "xdefs.h"
#include "mem_intrin.def"
int memcpy_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length); int memcpy_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length);
int memmove_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length); int memmove_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length);
int memset_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length); int memset_test(uint8_t *buf, uint8_t *buf2, uint8_t init, SizeT length);
int memcpy_test_fixed_len(uint8_t init); #define X(NBYTES) \
int memmove_test_fixed_len(uint8_t init); int memcpy_test_fixed_len_##NBYTES(uint8_t init); \
int memset_test_fixed_len(uint8_t init); int memmove_test_fixed_len_##NBYTES(uint8_t init); \
int memset_test_fixed_len_##NBYTES(uint8_t init);
MEMINTRIN_SIZE_TABLE
#undef X
crosstest/mem_intrin_main.cpp
...@@ -14,27 +14,6 @@ namespace Subzero_ { ...@@ -14,27 +14,6 @@ namespace Subzero_ {
#define XSTR(s) STR(s) #define XSTR(s) STR(s)
#define STR(s) #s #define STR(s) #s
void testFixedLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) {
#define do_test_fixed(test_func) \
for (uint8_t init_val = 0; init_val < 100; ++init_val) { \
++TotalTests; \
int llc_result = test_func(init_val); \
int sz_result = Subzero_::test_func(init_val); \
if (llc_result == sz_result) { \
++Passes; \
} else { \
++Failures; \
printf("Failure (%s): init_val=%d, llc=%d, sz=%d\n", STR(test_func), \
init_val, llc_result, sz_result); \
} \
}
do_test_fixed(memcpy_test_fixed_len);
do_test_fixed(memmove_test_fixed_len);
do_test_fixed(memset_test_fixed_len)
#undef do_test_fixed
}
void testVariableLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) { void testVariableLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) {
uint8_t buf[256]; uint8_t buf[256];
uint8_t buf2[256]; uint8_t buf2[256];
...@@ -60,6 +39,30 @@ void testVariableLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) { ...@@ -60,6 +39,30 @@ void testVariableLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) {
#undef do_test_variable #undef do_test_variable
} }
void testFixedLen(SizeT &TotalTests, SizeT &Passes, SizeT &Failures) {
#define do_test_fixed(test_func, NBYTES) \
for (uint8_t init_val = 0; init_val < 100; ++init_val) { \
++TotalTests; \
int llc_result = test_func##_##NBYTES(init_val); \
int sz_result = Subzero_::test_func##_##NBYTES(init_val); \
if (llc_result == sz_result) { \
++Passes; \
} else { \
++Failures; \
printf("Failure (%s): init_val=%d, len=%d, llc=%d, sz=%d\n", \
STR(test_func), init_val, NBYTES, llc_result, sz_result); \
} \
}
#define X(NBYTES) \
do_test_fixed(memcpy_test_fixed_len, NBYTES); \
do_test_fixed(memmove_test_fixed_len, NBYTES); \
do_test_fixed(memset_test_fixed_len, NBYTES);
MEMINTRIN_SIZE_TABLE
#undef X
#undef do_test_fixed
}
#ifdef X8664_STACK_HACK #ifdef X8664_STACK_HACK
extern "C" int wrapped_main(int argc, char *argv[]) { extern "C" int wrapped_main(int argc, char *argv[]) {
#else // !defined(X8664_STACK_HACK) #else // !defined(X8664_STACK_HACK)
......
pydir/crosstest.py
...@@ -93,6 +93,8 @@ def main(): ...@@ -93,6 +93,8 @@ def main():
argparser.add_argument('--filetype', default='obj', dest='filetype', argparser.add_argument('--filetype', default='obj', dest='filetype',
choices=['obj', 'asm', 'iasm'], choices=['obj', 'asm', 'iasm'],
help='Output file type. Default %(default)s.') help='Output file type. Default %(default)s.')
argparser.add_argument('--sz', dest='sz_args', action='append', default=[],
help='Extra arguments to pass to pnacl-sz.')
args = argparser.parse_args() args = argparser.parse_args()
nacl_root = FindBaseNaCl() nacl_root = FindBaseNaCl()
...@@ -133,6 +135,7 @@ def main(): ...@@ -133,6 +135,7 @@ def main():
obj_sz = os.path.join(args.dir, base_sz + '.sz.o') obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
obj_llc = os.path.join(args.dir, base_sz + '.llc.o') obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)), shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
] + args.sz_args + [
'-O' + args.optlevel, '-O' + args.optlevel,
'-mattr=' + args.attr, '-mattr=' + args.attr,
'--target=' + args.target, '--target=' + args.target,
......
src/IceAssemblerX86Base.h
...@@ -845,6 +845,8 @@ public: ...@@ -845,6 +845,8 @@ public:
void cmpxchg8b(const typename Traits::Address &address, bool Locked); void cmpxchg8b(const typename Traits::Address &address, bool Locked);
void xadd(Type Ty, const typename Traits::Address &address, void xadd(Type Ty, const typename Traits::Address &address,
typename Traits::GPRRegister reg, bool Locked); typename Traits::GPRRegister reg, bool Locked);
void xchg(Type Ty, typename Traits::GPRRegister reg0,
typename Traits::GPRRegister reg1);
void xchg(Type Ty, const typename Traits::Address &address, void xchg(Type Ty, const typename Traits::Address &address,
typename Traits::GPRRegister reg); typename Traits::GPRRegister reg);
......
src/IceAssemblerX86BaseImpl.h
...@@ -3150,6 +3150,29 @@ void AssemblerX86Base<Machine>::xadd(Type Ty, ...@@ -3150,6 +3150,29 @@ void AssemblerX86Base<Machine>::xadd(Type Ty,
} }
template <class Machine> template <class Machine>
void AssemblerX86Base<Machine>::xchg(Type Ty, typename Traits::GPRRegister reg0,
typename Traits::GPRRegister reg1) {
AssemblerBuffer::EnsureCapacity ensured(&Buffer);
if (Ty == IceType_i16)
emitOperandSizeOverride();
// Use short form if either register is EAX.
if (reg0 == Traits::Encoded_Reg_Accumulator) {
emitRexB(Ty, reg1);
emitUint8(0x90 + gprEncoding(reg1));
} else if (reg1 == Traits::Encoded_Reg_Accumulator) {
emitRexB(Ty, reg0);
emitUint8(0x90 + gprEncoding(reg0));
} else {
emitRexRB(Ty, reg0, reg1);
if (isByteSizedArithType(Ty))
emitUint8(0x86);
else
emitUint8(0x87);
emitRegisterOperand(gprEncoding(reg0), gprEncoding(reg1));
}
}
template <class Machine>
void AssemblerX86Base<Machine>::xchg(Type Ty, void AssemblerX86Base<Machine>::xchg(Type Ty,
const typename Traits::Address &addr, const typename Traits::Address &addr,
typename Traits::GPRRegister reg) { typename Traits::GPRRegister reg) {
......
src/IceClFlags.cpp
...@@ -83,6 +83,10 @@ cl::opt<bool> EnableBlockProfile( ...@@ -83,6 +83,10 @@ cl::opt<bool> EnableBlockProfile(
cl::init(false)); cl::init(false));
cl::opt<bool> cl::opt<bool>
ForceMemIntrinOpt("fmem-intrin-opt",
cl::desc("Force optimization of memory intrinsics."));
cl::opt<bool>
FunctionSections("ffunction-sections", FunctionSections("ffunction-sections",
cl::desc("Emit functions into separate sections")); cl::desc("Emit functions into separate sections"));
...@@ -356,6 +360,7 @@ void ClFlags::resetClFlags(ClFlags &OutFlags) { ...@@ -356,6 +360,7 @@ void ClFlags::resetClFlags(ClFlags &OutFlags) {
OutFlags.DisableTranslation = false; OutFlags.DisableTranslation = false;
OutFlags.DumpStats = false; OutFlags.DumpStats = false;
OutFlags.EnableBlockProfile = false; OutFlags.EnableBlockProfile = false;
OutFlags.ForceMemIntrinOpt = false;
OutFlags.FunctionSections = false; OutFlags.FunctionSections = false;
OutFlags.GenerateUnitTestMessages = false; OutFlags.GenerateUnitTestMessages = false;
OutFlags.PhiEdgeSplit = false; OutFlags.PhiEdgeSplit = false;
...@@ -416,6 +421,7 @@ void ClFlags::getParsedClFlags(ClFlags &OutFlags) { ...@@ -416,6 +421,7 @@ void ClFlags::getParsedClFlags(ClFlags &OutFlags) {
OutFlags.setDisableTranslation(::DisableTranslation); OutFlags.setDisableTranslation(::DisableTranslation);
OutFlags.setDumpStats(::DumpStats); OutFlags.setDumpStats(::DumpStats);
OutFlags.setEnableBlockProfile(::EnableBlockProfile); OutFlags.setEnableBlockProfile(::EnableBlockProfile);
OutFlags.setForceMemIntrinOpt(::ForceMemIntrinOpt);
OutFlags.setFunctionSections(::FunctionSections); OutFlags.setFunctionSections(::FunctionSections);
OutFlags.setNumTranslationThreads(::NumThreads); OutFlags.setNumTranslationThreads(::NumThreads);
OutFlags.setOptLevel(::OLevel); OutFlags.setOptLevel(::OLevel);
......
src/IceClFlags.h
...@@ -72,6 +72,9 @@ public: ...@@ -72,6 +72,9 @@ public:
bool getEnableBlockProfile() const { return EnableBlockProfile; } bool getEnableBlockProfile() const { return EnableBlockProfile; }
void setEnableBlockProfile(bool NewValue) { EnableBlockProfile = NewValue; } void setEnableBlockProfile(bool NewValue) { EnableBlockProfile = NewValue; }
bool getForceMemIntrinOpt() const { return ForceMemIntrinOpt; }
void setForceMemIntrinOpt(bool NewValue) { ForceMemIntrinOpt = NewValue; }
bool getFunctionSections() const { return FunctionSections; } bool getFunctionSections() const { return FunctionSections; }
void setFunctionSections(bool NewValue) { FunctionSections = NewValue; } void setFunctionSections(bool NewValue) { FunctionSections = NewValue; }
...@@ -241,6 +244,7 @@ private: ...@@ -241,6 +244,7 @@ private:
bool DisableTranslation; bool DisableTranslation;
bool DumpStats; bool DumpStats;
bool EnableBlockProfile; bool EnableBlockProfile;
bool ForceMemIntrinOpt;
bool FunctionSections; bool FunctionSections;
bool GenerateUnitTestMessages; bool GenerateUnitTestMessages;
bool PhiEdgeSplit; bool PhiEdgeSplit;
......
src/IceInstX86BaseImpl.h
...@@ -3202,19 +3202,29 @@ void InstX86Xchg<Machine>::emitIAS(const Cfg *Func) const { ...@@ -3202,19 +3202,29 @@ void InstX86Xchg<Machine>::emitIAS(const Cfg *Func) const {
typename InstX86Base<Machine>::Traits::Assembler *Asm = typename InstX86Base<Machine>::Traits::Assembler *Asm =
Func->getAssembler<typename InstX86Base<Machine>::Traits::Assembler>(); Func->getAssembler<typename InstX86Base<Machine>::Traits::Assembler>();
Type Ty = this->getSrc(0)->getType(); Type Ty = this->getSrc(0)->getType();
const auto Mem = const auto *VarReg1 = llvm::cast<Variable>(this->getSrc(1));
assert(VarReg1->hasReg());
const typename InstX86Base<Machine>::Traits::RegisterSet::GPRRegister Reg1 =
InstX86Base<Machine>::Traits::RegisterSet::getEncodedGPR(
VarReg1->getRegNum());
if (const auto *VarReg0 = llvm::dyn_cast<Variable>(this->getSrc(0))) {
assert(VarReg0->hasReg());
const typename InstX86Base<Machine>::Traits::RegisterSet::GPRRegister Reg0 =
InstX86Base<Machine>::Traits::RegisterSet::getEncodedGPR(
VarReg0->getRegNum());
Asm->xchg(Ty, Reg0, Reg1);
return;
}
const auto *Mem =
llvm::cast<typename InstX86Base<Machine>::Traits::X86OperandMem>( llvm::cast<typename InstX86Base<Machine>::Traits::X86OperandMem>(
this->getSrc(0)); this->getSrc(0));
assert(Mem->getSegmentRegister() == assert(Mem->getSegmentRegister() ==
InstX86Base<Machine>::Traits::X86OperandMem::DefaultSegment); InstX86Base<Machine>::Traits::X86OperandMem::DefaultSegment);
const typename InstX86Base<Machine>::Traits::Address Addr = const typename InstX86Base<Machine>::Traits::Address Addr =
Mem->toAsmAddress(Asm); Mem->toAsmAddress(Asm);
const auto VarReg = llvm::cast<Variable>(this->getSrc(1)); Asm->xchg(Ty, Addr, Reg1);
assert(VarReg->hasReg());
const typename InstX86Base<Machine>::Traits::RegisterSet::GPRRegister Reg =
InstX86Base<Machine>::Traits::RegisterSet::getEncodedGPR(
VarReg->getRegNum());
Asm->xchg(Ty, Addr, Reg);
} }
template <class Machine> template <class Machine>
......
src/IceTargetLowering.cpp
...@@ -440,6 +440,11 @@ InstCall *TargetLowering::makeHelperCall(const IceString &Name, Variable *Dest, ...@@ -440,6 +440,11 @@ InstCall *TargetLowering::makeHelperCall(const IceString &Name, Variable *Dest,
return Call; return Call;
} }
bool TargetLowering::shouldOptimizeMemIntrins() {
return Ctx->getFlags().getOptLevel() >= Opt_1 ||
Ctx->getFlags().getForceMemIntrinOpt();
}
void TargetLowering::emitWithoutPrefix(const ConstantRelocatable *C) const { void TargetLowering::emitWithoutPrefix(const ConstantRelocatable *C) const {
if (!BuildDefs::dump()) if (!BuildDefs::dump())
return; return;
......
src/IceTargetLowering.h
...@@ -351,6 +351,8 @@ protected: ...@@ -351,6 +351,8 @@ protected:
Context.getLastInserted()->setDestNonKillable(); Context.getLastInserted()->setDestNonKillable();
} }
bool shouldOptimizeMemIntrins();
Cfg *Func; Cfg *Func;
GlobalContext *Ctx; GlobalContext *Ctx;
bool HasComputedFrame = false; bool HasComputedFrame = false;
......
src/IceTargetLoweringX8632Traits.h
...@@ -441,6 +441,13 @@ template <> struct MachineTraits<TargetX8632> { ...@@ -441,6 +441,13 @@ template <> struct MachineTraits<TargetX8632> {
/// The number of different NOP instructions /// The number of different NOP instructions
static const uint32_t X86_NUM_NOP_VARIANTS = 5; static const uint32_t X86_NUM_NOP_VARIANTS = 5;
/// \name Limits for unrolling memory intrinsics.
/// @{
static constexpr uint32_t MEMCPY_UNROLL_LIMIT = 8;
static constexpr uint32_t MEMMOVE_UNROLL_LIMIT = 8;
static constexpr uint32_t MEMSET_UNROLL_LIMIT = 16;
/// @}
/// Value is in bytes. Return Value adjusted to the next highest multiple /// Value is in bytes. Return Value adjusted to the next highest multiple
/// of the stack alignment. /// of the stack alignment.
static uint32_t applyStackAlignment(uint32_t Value) { static uint32_t applyStackAlignment(uint32_t Value) {
......
src/IceTargetLoweringX8664Traits.h
...@@ -456,6 +456,13 @@ template <> struct MachineTraits<TargetX8664> { ...@@ -456,6 +456,13 @@ template <> struct MachineTraits<TargetX8664> {
/// The number of different NOP instructions /// The number of different NOP instructions
static const uint32_t X86_NUM_NOP_VARIANTS = 5; static const uint32_t X86_NUM_NOP_VARIANTS = 5;
/// \name Limits for unrolling memory intrinsics.
/// @{
static constexpr uint32_t MEMCPY_UNROLL_LIMIT = 8;
static constexpr uint32_t MEMMOVE_UNROLL_LIMIT = 8;
static constexpr uint32_t MEMSET_UNROLL_LIMIT = 16;
/// @}
/// Value is in bytes. Return Value adjusted to the next highest multiple /// Value is in bytes. Return Value adjusted to the next highest multiple
/// of the stack alignment. /// of the stack alignment.
static uint32_t applyStackAlignment(uint32_t Value) { static uint32_t applyStackAlignment(uint32_t Value) {
......
src/IceTargetLoweringX86Base.h
...@@ -192,9 +192,17 @@ protected: ...@@ -192,9 +192,17 @@ protected:
Operand *Val); Operand *Val);
void lowerCountZeros(bool Cttz, Type Ty, Variable *Dest, Operand *FirstVal, void lowerCountZeros(bool Cttz, Type Ty, Variable *Dest, Operand *FirstVal,
Operand *SecondVal); Operand *SecondVal);
/// Replace a call to memcpy with inline instructions. /// Load from memory for a given type.
void typedLoad(Type Ty, Variable *Dest, Variable *Base, Constant *Offset);
/// Store to memory for a given type.
void typedStore(Type Ty, Variable *Value, Variable *Base, Constant *Offset);
/// Copy memory of given type from Src to Dest using OffsetAmt on both.
void copyMemory(Type Ty, Variable *Dest, Variable *Src, int32_t OffsetAmt);
/// Replace some calls to memcpy with inline instructions.
void lowerMemcpy(Operand *Dest, Operand *Src, Operand *Count); void lowerMemcpy(Operand *Dest, Operand *Src, Operand *Count);
/// Replace a call to memset with inline instructions. /// Replace some calls to memmove with inline instructions.
void lowerMemmove(Operand *Dest, Operand *Src, Operand *Count);
/// Replace some calls to memset with inline instructions.
void lowerMemset(Operand *Dest, Operand *Val, Operand *Count); void lowerMemset(Operand *Dest, Operand *Val, Operand *Count);
/// Lower an indirect jump adding sandboxing when needed. /// Lower an indirect jump adding sandboxing when needed.
...@@ -251,6 +259,19 @@ protected: ...@@ -251,6 +259,19 @@ protected:
Variable *makeReg(Type Ty, int32_t RegNum = Variable::NoRegister); Variable *makeReg(Type Ty, int32_t RegNum = Variable::NoRegister);
static Type stackSlotType(); static Type stackSlotType();
static constexpr uint32_t NoSizeLimit = 0;
static const Type TypeForSize[];
/// Returns the largest type which is equal to or larger than Size bytes. The
/// type is suitable for copying memory i.e. a load and store will be a
/// single instruction (for example x86 will get f64 not i64).
static Type largestTypeInSize(uint32_t Size, uint32_t MaxSize = NoSizeLimit);
/// Returns the smallest type which is equal to or larger than Size bytes. If
/// one doesn't exist then the largest type smaller than Size bytes is
/// returned. The type is suitable for memory copies as described at
/// largestTypeInSize.
static Type firstTypeThatFitsSize(uint32_t Size,
uint32_t MaxSize = NoSizeLimit);
Variable *copyToReg(Operand *Src, int32_t RegNum = Variable::NoRegister); Variable *copyToReg(Operand *Src, int32_t RegNum = Variable::NoRegister);
/// \name Returns a vector in a register with the given constant entries. /// \name Returns a vector in a register with the given constant entries.
......
src/IceTargetLoweringX86BaseImpl.h
...@@ -3153,11 +3153,7 @@ void TargetX86Base<Machine>::lowerIntrinsicCall( ...@@ -3153,11 +3153,7 @@ void TargetX86Base<Machine>::lowerIntrinsicCall(
return; return;
} }
case Intrinsics::Memmove: { case Intrinsics::Memmove: {
InstCall *Call = makeHelperCall(H_call_memmove, nullptr, 3); lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2));
Call->addArg(Instr->getArg(0));
Call->addArg(Instr->getArg(1));
Call->addArg(Instr->getArg(2));
lowerCall(Call);
return; return;
} }
case Intrinsics::Memset: { case Intrinsics::Memset: {
...@@ -3600,22 +3596,55 @@ void TargetX86Base<Machine>::lowerCountZeros(bool Cttz, Type Ty, Variable *Dest, ...@@ -3600,22 +3596,55 @@ void TargetX86Base<Machine>::lowerCountZeros(bool Cttz, Type Ty, Variable *Dest,
} }
template <class Machine> template <class Machine>
void TargetX86Base<Machine>::typedLoad(Type Ty, Variable *Dest, Variable *Base,
Constant *Offset) {
auto *Mem = Traits::X86OperandMem::create(Func, Ty, Base, Offset);
if (isVectorType(Ty))
_movp(Dest, Mem);
else if (Ty == IceType_f64)
_movq(Dest, Mem);
else
_mov(Dest, Mem);
}
template <class Machine>
void TargetX86Base<Machine>::typedStore(Type Ty, Variable *Value,
Variable *Base, Constant *Offset) {
auto *Mem = Traits::X86OperandMem::create(Func, Ty, Base, Offset);
if (isVectorType(Ty))
_storep(Value, Mem);
else if (Ty == IceType_f64)
_storeq(Value, Mem);
else
_store(Value, Mem);
}
template <class Machine>
void TargetX86Base<Machine>::copyMemory(Type Ty, Variable *Dest, Variable *Src,
int32_t OffsetAmt) {
Constant *Offset = OffsetAmt ? Ctx->getConstantInt32(OffsetAmt) : nullptr;
// TODO(ascull): this or add nullptr test to _movp, _movq
Variable *Data = makeReg(Ty);
typedLoad(Ty, Data, Src, Offset);
typedStore(Ty, Data, Dest, Offset);
}
template <class Machine>
void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src, void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src,
Operand *Count) { Operand *Count) {
// There is a load and store for each chunk in the unroll // There is a load and store for each chunk in the unroll
constexpr uint32_t UNROLL_LIMIT = 8;
constexpr uint32_t BytesPerStorep = 16; constexpr uint32_t BytesPerStorep = 16;
constexpr uint32_t BytesPerStoreq = 8;
constexpr uint32_t BytesPerStorei32 = 4;
constexpr uint32_t BytesPerStorei16 = 2;
constexpr uint32_t BytesPerStorei8 = 1;
// Check if the operands are constants // Check if the operands are constants
const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count); const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count);
const bool IsCountConst = CountConst != nullptr; const bool IsCountConst = CountConst != nullptr;
const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0; const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0;
if (IsCountConst && CountValue <= BytesPerStorep * UNROLL_LIMIT) { if (shouldOptimizeMemIntrins() && IsCountConst &&
CountValue <= BytesPerStorep * Traits::MEMCPY_UNROLL_LIMIT) {
// Unlikely, but nothing to do if it does happen // Unlikely, but nothing to do if it does happen
if (CountValue == 0) if (CountValue == 0)
return; return;
...@@ -3623,85 +3652,106 @@ void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src, ...@@ -3623,85 +3652,106 @@ void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src,
Variable *SrcBase = legalizeToReg(Src); Variable *SrcBase = legalizeToReg(Src);
Variable *DestBase = legalizeToReg(Dest); Variable *DestBase = legalizeToReg(Dest);
auto lowerCopy = [this, DestBase, SrcBase](Type Ty, uint32_t OffsetAmt) { // Find the largest type that can be used and use it as much as possible in
Constant *Offset = OffsetAmt ? Ctx->getConstantInt32(OffsetAmt) : nullptr; // reverse order. Then handle any remainder with overlapping copies. Since
// TODO(ascull): this or add nullptr test to _movp, _movq // the remainder will be at the end, there will be reduced pressure on the
Variable *Data = makeReg(Ty); // memory unit as the accesses to the same memory are far apart.
Type Ty = largestTypeInSize(CountValue);
// TODO(ascull): is 64-bit better with vector or scalar movq? uint32_t TyWidth = typeWidthInBytes(Ty);
auto *SrcMem = Traits::X86OperandMem::create(Func, Ty, SrcBase, Offset);
if (isVectorType(Ty))
_movp(Data, SrcMem);
else if (Ty == IceType_f64)
_movq(Data, SrcMem);
else
_mov(Data, SrcMem);
auto *DestMem = Traits::X86OperandMem::create(Func, Ty, DestBase, Offset);
if (isVectorType(Ty))
_storep(Data, DestMem);
else if (Ty == IceType_f64)
_storeq(Data, DestMem);
else
_store(Data, DestMem);
};
// Lowers the assignment to the remaining bytes. Assumes the original size uint32_t RemainingBytes = CountValue;
// was large enough to allow for overlaps. int32_t Offset = (CountValue & ~(TyWidth - 1)) - TyWidth;
auto lowerLeftOvers = [this, lowerCopy, CountValue](uint32_t Size) { while (RemainingBytes >= TyWidth) {
if (Size > BytesPerStoreq) { copyMemory(Ty, DestBase, SrcBase, Offset);
lowerCopy(IceType_v16i8, CountValue - BytesPerStorep); RemainingBytes -= TyWidth;
} else if (Size > BytesPerStorei32) { Offset -= TyWidth;
lowerCopy(IceType_f64, CountValue - BytesPerStoreq);
} else if (Size > BytesPerStorei16) {
lowerCopy(IceType_i32, CountValue - BytesPerStorei32);
} else if (Size > BytesPerStorei8) {
lowerCopy(IceType_i16, CountValue - BytesPerStorei16);
} else if (Size == BytesPerStorei8) {
lowerCopy(IceType_i8, CountValue - BytesPerStorei8);
} }
};
if (CountValue >= BytesPerStorep) { if (RemainingBytes == 0)
// Use large vector operations
for (uint32_t N = CountValue & 0xFFFFFFF0; N != 0;) {
N -= BytesPerStorep;
lowerCopy(IceType_v16i8, N);
}
lowerLeftOvers(CountValue & 0xF);
return; return;
}
// Too small to use large vector operations so use small ones instead // Lower the remaining bytes. Adjust to larger types in order to make use
if (CountValue >= BytesPerStoreq) { // of overlaps in the copies.
lowerCopy(IceType_f64, 0); Type LeftOverTy = firstTypeThatFitsSize(RemainingBytes);
lowerLeftOvers(CountValue - BytesPerStoreq); Offset = CountValue - typeWidthInBytes(LeftOverTy);
copyMemory(LeftOverTy, DestBase, SrcBase, Offset);
return; return;
} }
// Too small for vector operations so use scalar ones // Fall back on a function call
if (CountValue >= BytesPerStorei32) { InstCall *Call = makeHelperCall(H_call_memcpy, nullptr, 3);
lowerCopy(IceType_i32, 0); Call->addArg(Dest);
lowerLeftOvers(CountValue - BytesPerStorei32); Call->addArg(Src);
return; Call->addArg(Count);
} lowerCall(Call);
}
template <class Machine>
void TargetX86Base<Machine>::lowerMemmove(Operand *Dest, Operand *Src,
Operand *Count) {
// There is a load and store for each chunk in the unroll
constexpr uint32_t BytesPerStorep = 16;
// Check if the operands are constants
const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count);
const bool IsCountConst = CountConst != nullptr;
const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0;
// 3 is the awkward size as it is too small for the vector or 32-bit if (shouldOptimizeMemIntrins() && IsCountConst &&
// operations and will not work with lowerLeftOvers as there is no valid CountValue <= BytesPerStorep * Traits::MEMMOVE_UNROLL_LIMIT) {
// overlap. // Unlikely, but nothing to do if it does happen
if (CountValue == 3) { if (CountValue == 0)
lowerCopy(IceType_i16, 0);
lowerCopy(IceType_i8, 2);
return; return;
Variable *SrcBase = legalizeToReg(Src);
Variable *DestBase = legalizeToReg(Dest);
std::tuple<Type, Constant *, Variable *>
Moves[Traits::MEMMOVE_UNROLL_LIMIT];
Constant *Offset;
Variable *Reg;
// Copy the data into registers as the source and destination could overlap
// so make sure not to clobber the memory. This also means overlapping moves
// can be used as we are taking a safe snapshot of the memory.
Type Ty = largestTypeInSize(CountValue);
uint32_t TyWidth = typeWidthInBytes(Ty);
uint32_t RemainingBytes = CountValue;
int32_t OffsetAmt = (CountValue & ~(TyWidth - 1)) - TyWidth;
size_t N = 0;
while (RemainingBytes >= TyWidth) {
assert(N <= Traits::MEMMOVE_UNROLL_LIMIT);
Offset = Ctx->getConstantInt32(OffsetAmt);
Reg = makeReg(Ty);
typedLoad(Ty, Reg, SrcBase, Offset);
RemainingBytes -= TyWidth;
OffsetAmt -= TyWidth;
Moves[N++] = std::make_tuple(Ty, Offset, Reg);
}
if (RemainingBytes != 0) {
// Lower the remaining bytes. Adjust to larger types in order to make use
// of overlaps in the copies.
assert(N <= Traits::MEMMOVE_UNROLL_LIMIT);
Ty = firstTypeThatFitsSize(RemainingBytes);
Offset = Ctx->getConstantInt32(CountValue - typeWidthInBytes(Ty));
Reg = makeReg(Ty);
typedLoad(Ty, Reg, SrcBase, Offset);
Moves[N++] = std::make_tuple(Ty, Offset, Reg);
}
// Copy the data out into the destination memory
for (size_t i = 0; i < N; ++i) {
std::tie(Ty, Offset, Reg) = Moves[i];
typedStore(Ty, Reg, DestBase, Offset);
} }
// 1 or 2 can be done in a single scalar copy
lowerLeftOvers(CountValue);
return; return;
} }
// Fall back on a function call // Fall back on a function call
InstCall *Call = makeHelperCall(H_call_memcpy, nullptr, 3); InstCall *Call = makeHelperCall(H_call_memmove, nullptr, 3);
Call->addArg(Dest); Call->addArg(Dest);
Call->addArg(Src); Call->addArg(Src);
Call->addArg(Count); Call->addArg(Count);
...@@ -3711,12 +3761,9 @@ void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src, ...@@ -3711,12 +3761,9 @@ void TargetX86Base<Machine>::lowerMemcpy(Operand *Dest, Operand *Src,
template <class Machine> template <class Machine>
void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val, void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val,
Operand *Count) { Operand *Count) {
constexpr uint32_t UNROLL_LIMIT = 16;
constexpr uint32_t BytesPerStorep = 16; constexpr uint32_t BytesPerStorep = 16;
constexpr uint32_t BytesPerStoreq = 8; constexpr uint32_t BytesPerStoreq = 8;
constexpr uint32_t BytesPerStorei32 = 4; constexpr uint32_t BytesPerStorei32 = 4;
constexpr uint32_t BytesPerStorei16 = 2;
constexpr uint32_t BytesPerStorei8 = 1;
assert(Val->getType() == IceType_i8); assert(Val->getType() == IceType_i8);
// Check if the operands are constants // Check if the operands are constants
...@@ -3734,11 +3781,11 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val, ...@@ -3734,11 +3781,11 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val,
// TODO(ascull): if the count is constant but val is not it would be possible // TODO(ascull): if the count is constant but val is not it would be possible
// to inline by spreading the value across 4 bytes and accessing subregs e.g. // to inline by spreading the value across 4 bytes and accessing subregs e.g.
// eax, ax and al. // eax, ax and al.
if (IsCountConst && IsValConst) { if (shouldOptimizeMemIntrins() && IsCountConst && IsValConst) {
Variable *Base = nullptr; Variable *Base = nullptr;
Variable *VecReg = nullptr;
const uint32_t SpreadValue = const uint32_t SpreadValue =
(ValValue << 24) | (ValValue << 16) | (ValValue << 8) | ValValue; (ValValue << 24) | (ValValue << 16) | (ValValue << 8) | ValValue;
Variable *VecReg = nullptr;
auto lowerSet = [this, &Base, SpreadValue, &VecReg](Type Ty, auto lowerSet = [this, &Base, SpreadValue, &VecReg](Type Ty,
uint32_t OffsetAmt) { uint32_t OffsetAmt) {
...@@ -3750,7 +3797,7 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val, ...@@ -3750,7 +3797,7 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val,
if (isVectorType(Ty)) { if (isVectorType(Ty)) {
assert(VecReg != nullptr); assert(VecReg != nullptr);
_storep(VecReg, Mem); _storep(VecReg, Mem);
} else if (Ty == IceType_i64) { } else if (Ty == IceType_f64) {
assert(VecReg != nullptr); assert(VecReg != nullptr);
_storeq(VecReg, Mem); _storeq(VecReg, Mem);
} else { } else {
...@@ -3758,63 +3805,45 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val, ...@@ -3758,63 +3805,45 @@ void TargetX86Base<Machine>::lowerMemset(Operand *Dest, Operand *Val,
} }
}; };
// Lowers the assignment to the remaining bytes. Assumes the original size // Find the largest type that can be used and use it as much as possible in
// was large enough to allow for overlaps. // reverse order. Then handle any remainder with overlapping copies. Since
auto lowerLeftOvers = [this, lowerSet, CountValue](uint32_t Size) { // the remainder will be at the end, there will be reduces pressure on the
if (Size > BytesPerStoreq) { // memory unit as the access to the same memory are far apart.
lowerSet(IceType_v16i8, CountValue - BytesPerStorep); Type Ty;
} else if (Size > BytesPerStorei32) { if (ValValue == 0 && CountValue >= BytesPerStoreq &&
lowerSet(IceType_i64, CountValue - BytesPerStoreq); CountValue <= BytesPerStorep * Traits::MEMCPY_UNROLL_LIMIT) {
} else if (Size > BytesPerStorei16) {
lowerSet(IceType_i32, CountValue - BytesPerStorei32);
} else if (Size > BytesPerStorei8) {
lowerSet(IceType_i16, CountValue - BytesPerStorei16);
} else if (Size == BytesPerStorei8) {
lowerSet(IceType_i8, CountValue - BytesPerStorei8);
}
};
// When the value is zero it can be loaded into a vector register cheaply // When the value is zero it can be loaded into a vector register cheaply
// using the xor trick. // using the xor trick.
if (ValValue == 0 && CountValue >= BytesPerStoreq &&
CountValue <= BytesPerStorep * UNROLL_LIMIT) {
Base = legalizeToReg(Dest); Base = legalizeToReg(Dest);
VecReg = makeVectorOfZeros(IceType_v16i8); VecReg = makeVectorOfZeros(IceType_v16i8);
Ty = largestTypeInSize(CountValue);
// Too small to use large vector operations so use small ones instead } else if (CountValue <= BytesPerStorei32 * Traits::MEMCPY_UNROLL_LIMIT) {
if (CountValue < BytesPerStorep) { // When the value is non-zero or the count is small we can't use vector
lowerSet(IceType_i64, 0); // instructions so are limited to 32-bit stores.
lowerLeftOvers(CountValue - BytesPerStoreq); Base = legalizeToReg(Dest);
return; constexpr uint32_t MaxSize = 4;
Ty = largestTypeInSize(CountValue, MaxSize);
} }
// Use large vector operations if (Base) {
for (uint32_t N = CountValue & 0xFFFFFFF0; N != 0;) { uint32_t TyWidth = typeWidthInBytes(Ty);
N -= 16;
lowerSet(IceType_v16i8, N); uint32_t RemainingBytes = CountValue;
} uint32_t Offset = (CountValue & ~(TyWidth - 1)) - TyWidth;
lowerLeftOvers(CountValue & 0xF); while (RemainingBytes >= TyWidth) {
return; lowerSet(Ty, Offset);
RemainingBytes -= TyWidth;
Offset -= TyWidth;
} }
// TODO(ascull): load val into reg and select subregs e.g. eax, ax, al? if (RemainingBytes == 0)
if (CountValue <= BytesPerStorei32 * UNROLL_LIMIT) {
Base = legalizeToReg(Dest);
// 3 is the awkward size as it is too small for the vector or 32-bit
// operations and will not work with lowerLeftOvers as there is no valid
// overlap.
if (CountValue == 3) {
lowerSet(IceType_i16, 0);
lowerSet(IceType_i8, 2);
return; return;
}
// TODO(ascull); 64-bit can do better with 64-bit mov // Lower the remaining bytes. Adjust to larger types in order to make use
for (uint32_t N = CountValue & 0xFFFFFFFC; N != 0;) { // of overlaps in the copies.
N -= 4; Type LeftOverTy = firstTypeThatFitsSize(RemainingBytes);
lowerSet(IceType_i32, N); Offset = CountValue - typeWidthInBytes(LeftOverTy);
} lowerSet(LeftOverTy, Offset);
lowerLeftOvers(CountValue & 0x3);
return; return;
} }
} }
...@@ -5053,6 +5082,34 @@ Variable *TargetX86Base<Machine>::makeReg(Type Type, int32_t RegNum) { ...@@ -5053,6 +5082,34 @@ Variable *TargetX86Base<Machine>::makeReg(Type Type, int32_t RegNum) {
return Reg; return Reg;
} }
template <class Machine>
const Type TargetX86Base<Machine>::TypeForSize[] = {
IceType_i8, IceType_i16, IceType_i32,
(Traits::Is64Bit ? IceType_i64 : IceType_f64), IceType_v16i8};
template <class Machine>
Type TargetX86Base<Machine>::largestTypeInSize(uint32_t Size,
uint32_t MaxSize) {
assert(Size != 0);
uint32_t TyIndex = llvm::findLastSet(Size, llvm::ZB_Undefined);
uint32_t MaxIndex = MaxSize == NoSizeLimit
? llvm::array_lengthof(TypeForSize) - 1
: llvm::findLastSet(MaxSize, llvm::ZB_Undefined);
return TypeForSize[std::min(TyIndex, MaxIndex)];
}
template <class Machine>
Type TargetX86Base<Machine>::firstTypeThatFitsSize(uint32_t Size,
uint32_t MaxSize) {
assert(Size != 0);
uint32_t TyIndex = llvm::findLastSet(Size, llvm::ZB_Undefined);
if (!llvm::isPowerOf2_32(Size))
++TyIndex;
uint32_t MaxIndex = MaxSize == NoSizeLimit
? llvm::array_lengthof(TypeForSize) - 1
: llvm::findLastSet(MaxSize, llvm::ZB_Undefined);
return TypeForSize[std::min(TyIndex, MaxIndex)];
}
template <class Machine> void TargetX86Base<Machine>::postLower() { template <class Machine> void TargetX86Base<Machine>::postLower() {
if (Ctx->getFlags().getOptLevel() == Opt_m1) if (Ctx->getFlags().getOptLevel() == Opt_m1)
return; return;
......
tests_lit/llvm2ice_tests/nacl-mem-intrinsics.ll
...@@ -4,8 +4,11 @@ ...@@ -4,8 +4,11 @@
; RUN: --target x8632 -i %s --args -O2 -sandbox \ ; RUN: --target x8632 -i %s --args -O2 -sandbox \
; RUN: | %if --need=target_X8632 --command FileCheck %s ; RUN: | %if --need=target_X8632 --command FileCheck %s
; RUN: %if --need=target_X8632 --command %p2i --filetype=obj --disassemble \ ; RUN: %if --need=target_X8632 --command %p2i --filetype=obj --disassemble \
; RUN: --target x8632 -i %s --args -Om1 -sandbox \ ; RUN: --target x8632 -i %s --args -Om1 --fmem-intrin-opt -sandbox \
; RUN: | %if --need=target_X8632 --command FileCheck %s ; RUN: | %if --need=target_X8632 --command FileCheck %s
; RUN: %if --need=target_X8632 --command %p2i --filetype=obj --disassemble \
; RUN: --target x8632 -i %s --args -Om1 -sandbox \
; RUN: | %if --need=target_X8632 --command FileCheck --check-prefix OM1 %s
; RUN: %if --need=target_ARM32 --need=allow_dump \ ; RUN: %if --need=target_ARM32 --need=allow_dump \
; RUN: --command %p2i --filetype=asm --assemble --disassemble --target arm32 \ ; RUN: --command %p2i --filetype=asm --assemble --disassemble --target arm32 \
...@@ -27,6 +30,8 @@ entry: ...@@ -27,6 +30,8 @@ entry:
} }
; CHECK-LABEL: test_memcpy ; CHECK-LABEL: test_memcpy
; CHECK: call {{.*}} R_{{.*}} memcpy ; CHECK: call {{.*}} R_{{.*}} memcpy
; OM1-LABEL: test_memcpy
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy ; ARM32-LABEL: test_memcpy
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
...@@ -40,6 +45,8 @@ entry: ...@@ -40,6 +45,8 @@ entry:
} }
; CHECK-LABEL: test_memcpy_long_const_len ; CHECK-LABEL: test_memcpy_long_const_len
; CHECK: call {{.*}} R_{{.*}} memcpy ; CHECK: call {{.*}} R_{{.*}} memcpy
; OM1-LABEL: test_memcpy_long_const_len
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_long_const_len ; ARM32-LABEL: test_memcpy_long_const_len
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
...@@ -55,6 +62,8 @@ entry: ...@@ -55,6 +62,8 @@ entry:
; CHECK: mov [[REG:[^,]*]],WORD PTR [{{.*}}] ; CHECK: mov [[REG:[^,]*]],WORD PTR [{{.*}}]
; CHECK-NEXT: mov WORD PTR [{{.*}}],[[REG]] ; CHECK-NEXT: mov WORD PTR [{{.*}}],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memcpy_very_small_const_len
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_very_small_const_len ; ARM32-LABEL: test_memcpy_very_small_const_len
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
...@@ -72,6 +81,8 @@ entry: ...@@ -72,6 +81,8 @@ entry:
; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x2] ; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x2]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x2],[[REG]] ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x2],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memcpy_const_len_3
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_const_len_3 ; ARM32-LABEL: test_memcpy_const_len_3
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
...@@ -89,6 +100,8 @@ entry: ...@@ -89,6 +100,8 @@ entry:
; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x8] ; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x8]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x8],[[REG]] ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x8],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memcpy_mid_const_len
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_mid_const_len ; ARM32-LABEL: test_memcpy_mid_const_len
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
...@@ -106,10 +119,12 @@ entry: ...@@ -106,10 +119,12 @@ entry:
; CHECK-NEXT: movq [[REG:xmm[0-9]+]],QWORD PTR [{{.*}}+0x7] ; CHECK-NEXT: movq [[REG:xmm[0-9]+]],QWORD PTR [{{.*}}+0x7]
; CHECK-NEXT: movq QWORD PTR [{{.*}}+0x7],[[REG]] ; CHECK-NEXT: movq QWORD PTR [{{.*}}+0x7],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memcpy_mid_const_len_overlap
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_mid_const_len_overlap ; ARM32-LABEL: test_memcpy_mid_const_len_overlap
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
define void @test_memcpy_large_const_len_overlap(i32 %iptr_dst, i32 %iptr_src) { define void @test_memcpy_big_const_len_overlap(i32 %iptr_dst, i32 %iptr_src) {
entry: entry:
%dst = inttoptr i32 %iptr_dst to i8* %dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8* %src = inttoptr i32 %iptr_src to i8*
...@@ -117,13 +132,15 @@ entry: ...@@ -117,13 +132,15 @@ entry:
i32 30, i32 1, i1 false) i32 30, i32 1, i1 false)
ret void ret void
} }
; CHECK-LABEL: test_memcpy_large_const_len_overlap ; CHECK-LABEL: test_memcpy_big_const_len_overlap
; CHECK: movups [[REG:xmm[0-9]+]],XMMWORD PTR [{{.*}}] ; CHECK: movups [[REG:xmm[0-9]+]],XMMWORD PTR [{{.*}}]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[REG]] ; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[REG]]
; CHECK-NEXT: movups [[REG:xmm[0-9]+]],XMMWORD PTR [{{.*}}+0xe] ; CHECK-NEXT: movups [[REG:xmm[0-9]+]],XMMWORD PTR [{{.*}}+0xe]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0xe],[[REG]] ; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0xe],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; ARM32-LABEL: test_memcpy_large_const_len_overlap ; OM1-LABEL: test_memcpy_big_const_len_overlap
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_big_const_len_overlap
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
define void @test_memcpy_large_const_len(i32 %iptr_dst, i32 %iptr_src) { define void @test_memcpy_large_const_len(i32 %iptr_dst, i32 %iptr_src) {
...@@ -142,11 +159,11 @@ entry: ...@@ -142,11 +159,11 @@ entry:
; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x20] ; CHECK-NEXT: mov [[REG:[^,]*]],BYTE PTR [{{.*}}+0x20]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x20],[[REG]] ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x20],[[REG]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memcpy_large_const_len
; OM1: call {{.*}} memcpy
; ARM32-LABEL: test_memcpy_large_const_len ; ARM32-LABEL: test_memcpy_large_const_len
; ARM32: bl {{.*}} memcpy ; ARM32: bl {{.*}} memcpy
; TODO(jvoung) -- if we want to be clever, we can do memset without a function
; call similar to memcpy.
define void @test_memmove(i32 %iptr_dst, i32 %iptr_src, i32 %len) { define void @test_memmove(i32 %iptr_dst, i32 %iptr_src, i32 %len) {
entry: entry:
%dst = inttoptr i32 %iptr_dst to i8* %dst = inttoptr i32 %iptr_dst to i8*
...@@ -157,20 +174,138 @@ entry: ...@@ -157,20 +174,138 @@ entry:
} }
; CHECK-LABEL: test_memmove ; CHECK-LABEL: test_memmove
; CHECK: call {{.*}} R_{{.*}} memmove ; CHECK: call {{.*}} R_{{.*}} memmove
; OM1-LABEL: test_memmove
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove ; ARM32-LABEL: test_memmove
; ARM32: bl {{.*}} memmove ; ARM32: bl {{.*}} memmove
define void @test_memmove_const_len_align(i32 %iptr_dst, i32 %iptr_src) { define void @test_memmove_long_const_len(i32 %iptr_dst, i32 %iptr_src) {
entry: entry:
%dst = inttoptr i32 %iptr_dst to i8* %dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8* %src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src, call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 32, i32 1, i1 false) i32 4876, i32 1, i1 false)
ret void ret void
} }
; CHECK-LABEL: test_memmove_const_len_align ; CHECK-LABEL: test_memmove_long_const_len
; CHECK: call {{.*}} R_{{.*}} memmove ; CHECK: call {{.*}} R_{{.*}} memmove
; ARM32-LABEL: test_memmove_const_len_align ; OM1-LABEL: test_memmove_long_const_len
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_long_const_len
; ARM32: bl {{.*}} memmove
define void @test_memmove_very_small_const_len(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 2, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_very_small_const_len
; CHECK: mov [[REG:[^,]*]],WORD PTR [{{.*}}]
; CHECK-NEXT: mov WORD PTR [{{.*}}],[[REG]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_very_small_const_len
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_very_small_const_len
; ARM32: bl {{.*}} memmove
define void @test_memmove_const_len_3(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 3, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_const_len_3
; CHECK: mov [[REG0:[^,]*]],WORD PTR [{{.*}}]
; CHECK-NEXT: mov [[REG1:[^,]*]],BYTE PTR [{{.*}}+0x2]
; CHECK-NEXT: mov WORD PTR [{{.*}}],[[REG0]]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x2],[[REG1]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_const_len_3
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_const_len_3
; ARM32: bl {{.*}} memmove
define void @test_memmove_mid_const_len(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 9, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_mid_const_len
; CHECK: movq [[REG0:xmm[0-9]+]],QWORD PTR [{{.*}}]
; CHECK-NEXT: mov [[REG1:[^,]*]],BYTE PTR [{{.*}}+0x8]
; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[REG0]]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x8],[[REG1]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_mid_const_len
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_mid_const_len
; ARM32: bl {{.*}} memmove
define void @test_memmove_mid_const_len_overlap(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 15, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_mid_const_len_overlap
; CHECK: movq [[REG0:xmm[0-9]+]],QWORD PTR [{{.*}}]
; CHECK-NEXT: movq [[REG1:xmm[0-9]+]],QWORD PTR [{{.*}}+0x7]
; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[REG0]]
; CHECK-NEXT: movq QWORD PTR [{{.*}}+0x7],[[REG1]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_mid_const_len_overlap
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_mid_const_len_overlap
; ARM32: bl {{.*}} memmove
define void @test_memmove_big_const_len_overlap(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 30, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_big_const_len_overlap
; CHECK: movups [[REG0:xmm[0-9]+]],XMMWORD PTR [{{.*}}]
; CHECK-NEXT: movups [[REG1:xmm[0-9]+]],XMMWORD PTR [{{.*}}+0xe]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[REG0]]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0xe],[[REG1]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_big_const_len_overlap
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_big_const_len_overlap
; ARM32: bl {{.*}} memmove
define void @test_memmove_large_const_len(i32 %iptr_dst, i32 %iptr_src) {
entry:
%dst = inttoptr i32 %iptr_dst to i8*
%src = inttoptr i32 %iptr_src to i8*
call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src,
i32 33, i32 1, i1 false)
ret void
}
; CHECK-LABEL: test_memmove_large_const_len
; CHECK: movups [[REG0:xmm[0-9]+]],XMMWORD PTR [{{.*}}+0x10]
; CHECK-NEXT: movups [[REG1:xmm[0-9]+]],XMMWORD PTR [{{.*}}]
; CHECK-NEXT: mov [[REG2:[^,]*]],BYTE PTR [{{.*}}+0x20]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0x10],[[REG0]]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[REG1]]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x20],[[REG2]]
; CHECK-NOT: mov
; OM1-LABEL: test_memmove_large_const_len
; OM1: call {{.*}} memmove
; ARM32-LABEL: test_memmove_large_const_len
; ARM32: bl {{.*}} memmove ; ARM32: bl {{.*}} memmove
define void @test_memset(i32 %iptr_dst, i32 %wide_val, i32 %len) { define void @test_memset(i32 %iptr_dst, i32 %wide_val, i32 %len) {
...@@ -184,6 +319,9 @@ entry: ...@@ -184,6 +319,9 @@ entry:
; CHECK-LABEL: test_memset ; CHECK-LABEL: test_memset
; CHECK: movzx ; CHECK: movzx
; CHECK: call {{.*}} R_{{.*}} memset ; CHECK: call {{.*}} R_{{.*}} memset
; OM1-LABEL: test_memset
; OM1: movzx
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset ; ARM32-LABEL: test_memset
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -199,6 +337,9 @@ entry: ...@@ -199,6 +337,9 @@ entry:
; CHECK-LABEL: test_memset_const_len_align ; CHECK-LABEL: test_memset_const_len_align
; CHECK: movzx ; CHECK: movzx
; CHECK: call {{.*}} R_{{.*}} memset ; CHECK: call {{.*}} R_{{.*}} memset
; OM1-LABEL: test_memset_const_len_align
; OM1: movzx
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_const_len_align ; ARM32-LABEL: test_memset_const_len_align
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -212,6 +353,8 @@ entry: ...@@ -212,6 +353,8 @@ entry:
} }
; CHECK-LABEL: test_memset_long_const_len_zero_val_align ; CHECK-LABEL: test_memset_long_const_len_zero_val_align
; CHECK: call {{.*}} R_{{.*}} memset ; CHECK: call {{.*}} R_{{.*}} memset
; OM1-LABEL: test_memset_long_const_len_zero_val_align
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_long_const_len_zero_val_align ; ARM32-LABEL: test_memset_long_const_len_zero_val_align
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -225,6 +368,8 @@ entry: ...@@ -225,6 +368,8 @@ entry:
; CHECK-LABEL: test_memset_const_val ; CHECK-LABEL: test_memset_const_val
; CHECK-NOT: movzx ; CHECK-NOT: movzx
; CHECK: call {{.*}} R_{{.*}} memset ; CHECK: call {{.*}} R_{{.*}} memset
; OM1-LABEL: test_memset_const_val
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_const_val ; ARM32-LABEL: test_memset_const_val
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -238,6 +383,8 @@ entry: ...@@ -238,6 +383,8 @@ entry:
; CHECK-LABEL: test_memset_const_val_len_very_small ; CHECK-LABEL: test_memset_const_val_len_very_small
; CHECK: mov WORD PTR [{{.*}}],0xa0a ; CHECK: mov WORD PTR [{{.*}}],0xa0a
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_const_val_len_very_small
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_const_val_len_very_small ; ARM32-LABEL: test_memset_const_val_len_very_small
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -252,6 +399,8 @@ entry: ...@@ -252,6 +399,8 @@ entry:
; CHECK: mov WORD PTR [{{.*}}],0x1010 ; CHECK: mov WORD PTR [{{.*}}],0x1010
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x2],0x10 ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x2],0x10
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_const_val_len_3
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_const_val_len_3 ; ARM32-LABEL: test_memset_const_val_len_3
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -267,6 +416,8 @@ entry: ...@@ -267,6 +416,8 @@ entry:
; CHECK: mov DWORD PTR [{{.*}}],0x20202020 ; CHECK: mov DWORD PTR [{{.*}}],0x20202020
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x8],0x20 ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x8],0x20
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_const_val_len_mid
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_const_val_len_mid ; ARM32-LABEL: test_memset_const_val_len_mid
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -282,6 +433,8 @@ entry: ...@@ -282,6 +433,8 @@ entry:
; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[ZERO]] ; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[ZERO]]
; CHECK-NEXT: mov DWORD PTR [{{.*}}+0x8],0x0 ; CHECK-NEXT: mov DWORD PTR [{{.*}}+0x8],0x0
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_zero_const_len_small
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_zero_const_len_small ; ARM32-LABEL: test_memset_zero_const_len_small
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -297,22 +450,26 @@ entry: ...@@ -297,22 +450,26 @@ entry:
; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[ZERO]] ; CHECK-NEXT: movq QWORD PTR [{{.*}}],[[ZERO]]
; CHECK-NEXT: movq QWORD PTR [{{.*}}+0x7],[[ZERO]] ; CHECK-NEXT: movq QWORD PTR [{{.*}}+0x7],[[ZERO]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_zero_const_len_small_overlap
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_zero_const_len_small_overlap ; ARM32-LABEL: test_memset_zero_const_len_small_overlap
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
define void @test_memset_zero_const_len_large_overlap(i32 %iptr_dst) { define void @test_memset_zero_const_len_big_overlap(i32 %iptr_dst) {
entry: entry:
%dst = inttoptr i32 %iptr_dst to i8* %dst = inttoptr i32 %iptr_dst to i8*
call void @llvm.memset.p0i8.i32(i8* %dst, i8 0, i32 30, i32 1, i1 false) call void @llvm.memset.p0i8.i32(i8* %dst, i8 0, i32 30, i32 1, i1 false)
ret void ret void
} }
; CHECK-LABEL: test_memset_zero_const_len_large_overlap ; CHECK-LABEL: test_memset_zero_const_len_big_overlap
; CHECK: pxor [[ZERO:xmm[0-9]+]],[[ZERO]] ; CHECK: pxor [[ZERO:xmm[0-9]+]],[[ZERO]]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[ZERO]] ; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[ZERO]]
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0xe],[[ZERO]] ; CHECK-NEXT: movups XMMWORD PTR [{{.*}}+0xe],[[ZERO]]
; CHECK-NOT: mov ; CHECK-NOT: mov
; ARM32-LABEL: test_memset_zero_const_len_large_overlap ; OM1-LABEL: test_memset_zero_const_len_big_overlap
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_zero_const_len_big_overlap
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
...@@ -328,6 +485,8 @@ entry: ...@@ -328,6 +485,8 @@ entry:
; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[ZERO]] ; CHECK-NEXT: movups XMMWORD PTR [{{.*}}],[[ZERO]]
; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x20],0x0 ; CHECK-NEXT: mov BYTE PTR [{{.*}}+0x20],0x0
; CHECK-NOT: mov ; CHECK-NOT: mov
; OM1-LABEL: test_memset_zero_const_len_large
; OM1: call {{.*}} R_{{.*}} memset
; ARM32-LABEL: test_memset_zero_const_len_large ; ARM32-LABEL: test_memset_zero_const_len_large
; ARM32: uxtb ; ARM32: uxtb
; ARM32: bl {{.*}} memset ; ARM32: bl {{.*}} memset
unittest/AssemblerX8632/Locked.cpp
...@@ -82,6 +82,57 @@ TEST_F(AssemblerX8632Test, Xchg) { ...@@ -82,6 +82,57 @@ TEST_F(AssemblerX8632Test, Xchg) {
#undef TestImpl #undef TestImpl
#undef TestImplSize #undef TestImplSize
#undef TestImplAddrReg #undef TestImplAddrReg
#define TestImplRegReg(Reg0, Value0, Reg1, Value1, Size) \
do { \
static constexpr char TestString[] = \
"(" #Reg0 "," #Value0 ", " #Reg1 ", " #Value1 ", " #Size ")"; \
const uint32_t V0 = (Value0)&Mask##Size; \
const uint32_t V1 = (Value1)&Mask##Size; \
\
__ mov(IceType_i##Size, GPRRegister::Encoded_Reg_##Reg0, \
Immediate(Value0)); \
__ mov(IceType_i##Size, GPRRegister::Encoded_Reg_##Reg1, \
Immediate(Value1)); \
__ xchg(IceType_i##Size, GPRRegister::Encoded_Reg_##Reg0, \
GPRRegister::Encoded_Reg_##Reg1); \
__ And(IceType_i32, GPRRegister::Encoded_Reg_##Reg0, \
Immediate(Mask##Size)); \
__ And(IceType_i32, GPRRegister::Encoded_Reg_##Reg1, \
Immediate(Mask##Size)); \
\
AssembledTest test = assemble(); \
test.run(); \
\
ASSERT_EQ(V0, test.Reg1()) << TestString; \
ASSERT_EQ(V1, test.Reg0()) << TestString; \
reset(); \
} while (0)
#define TestImplSize(Reg0, Reg1, Size) \
do { \
TestImplRegReg(Reg0, 0xa2b34567, Reg1, 0x0507ddee, Size); \
} while (0)
#define TestImpl(Reg0, Reg1) \
do { \
if (GPRRegister::Encoded_Reg_##Reg0 < 4 && \
GPRRegister::Encoded_Reg_##Reg1 < 4) { \
TestImplSize(Reg0, Reg1, 8); \
} \
TestImplSize(Reg0, Reg1, 16); \
TestImplSize(Reg0, Reg1, 32); \
} while (0)
TestImpl(eax, ebx);
TestImpl(edx, eax);
TestImpl(ecx, edx);
TestImpl(esi, eax);
TestImpl(edx, edi);
#undef TestImpl
#undef TestImplSize
#undef TestImplRegReg
} }
TEST_F(AssemblerX8632Test, Xadd) { TEST_F(AssemblerX8632Test, Xadd) {
......
unittest/AssemblerX8664/Locked.cpp
...@@ -85,6 +85,54 @@ TEST_F(AssemblerX8664Test, Xchg) { ...@@ -85,6 +85,54 @@ TEST_F(AssemblerX8664Test, Xchg) {
#undef TestImpl #undef TestImpl
#undef TestImplSize #undef TestImplSize
#undef TestImplAddrReg #undef TestImplAddrReg
#define TestImplRegReg(Reg0, Value0, Reg1, Value1, Size) \
do { \
static constexpr char TestString[] = \
"(" #Reg0 "," #Value0 ", " #Reg1 ", " #Value1 ", " #Size ")"; \
const uint32_t V0 = (Value0)&Mask##Size; \
const uint32_t V1 = (Value1)&Mask##Size; \
\
__ mov(IceType_i##Size, Encoded_GPR_##Reg0(), Immediate(Value0)); \
__ mov(IceType_i##Size, Encoded_GPR_##Reg1(), Immediate(Value1)); \
__ xchg(IceType_i##Size, Encoded_GPR_##Reg0(), Encoded_GPR_##Reg1()); \
__ And(IceType_i32, Encoded_GPR_##Reg0(), Immediate(Mask##Size)); \
__ And(IceType_i32, Encoded_GPR_##Reg1(), Immediate(Mask##Size)); \
\
AssembledTest test = assemble(); \
test.run(); \
\
ASSERT_EQ(V0, test.Reg1()) << TestString; \
ASSERT_EQ(V1, test.Reg0()) << TestString; \
reset(); \
} while (0)
#define TestImplSize(Reg0, Reg1, Size) \
do { \
TestImplRegReg(Reg0, 0xa2b34567, Reg1, 0x0507ddee, Size); \
} while (0)
#define TestImpl(Reg0, Reg1) \
do { \
TestImplSize(Reg0, Reg1, 8); \
TestImplSize(Reg0, Reg1, 16); \
TestImplSize(Reg0, Reg1, 32); \
} while (0)
// r1 == rax so has a short encoding
TestImpl(r6, r1);
TestImpl(r1, r8);
TestImpl(r2, r10);
TestImpl(r3, r11);
TestImpl(r4, r12);
TestImpl(r5, r13);
TestImpl(r6, r14);
TestImpl(r7, r15);
#undef TestImpl
#undef TestImplSize
#undef TestImplRegReg
} }
TEST_F(AssemblerX8664Test, Xadd) { TEST_F(AssemblerX8664Test, Xadd) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment