42#define DEBUG_TYPE "legalizer"
45using namespace LegalizeActions;
46using namespace MIPatternMatch;
55static std::pair<int, int>
61 unsigned NumParts =
Size / NarrowSize;
62 unsigned LeftoverSize =
Size - NumParts * NarrowSize;
65 if (LeftoverSize == 0)
70 if (LeftoverSize % EltSize != 0)
80 return std::make_pair(NumParts, NumLeftover);
107 : MIRBuilder(Builder), Observer(Observer),
MRI(MF.getRegInfo()),
108 LI(*MF.getSubtarget().getLegalizerInfo()),
109 TLI(*MF.getSubtarget().getTargetLowering()), KB(nullptr) {}
114 : MIRBuilder(
B), Observer(Observer),
MRI(MF.getRegInfo()), LI(LI),
115 TLI(*MF.getSubtarget().getTargetLowering()), KB(KB) {}
124 if (isa<GIntrinsic>(
MI))
127 switch (Step.Action) {
142 return bitcast(
MI, Step.TypeIdx, Step.NewType);
145 return lower(
MI, Step.TypeIdx, Step.NewType);
162void LegalizerHelper::insertParts(
Register DstReg,
184 assert(LeftoverRegs.
size() == 1 &&
"Expected one leftover register");
186 for (
auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
188 return mergeMixedSubvectors(DstReg, AllRegs);
193 for (
auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
194 extractGCDType(GCDRegs, GCDTy, PartReg);
195 LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs);
196 buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
209void LegalizerHelper::mergeMixedSubvectors(
Register DstReg,
212 for (
unsigned i = 0; i < PartRegs.
size() - 1; ++i)
213 appendVectorElts(AllElts, PartRegs[i]);
219 appendVectorElts(AllElts, Leftover);
227 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
229 const int StartIdx = Regs.
size();
230 const int NumResults =
MI.getNumOperands() - 1;
232 for (
int I = 0;
I != NumResults; ++
I)
233 Regs[StartIdx +
I] =
MI.getOperand(
I).getReg();
239 if (SrcTy == GCDTy) {
254 extractGCDType(Parts, GCDTy, SrcReg);
258LLT LegalizerHelper::buildLCMMergePieces(
LLT DstTy,
LLT NarrowTy,
LLT GCDTy,
260 unsigned PadStrategy) {
265 int NumOrigSrc = VRegs.
size();
271 if (NumOrigSrc < NumParts * NumSubParts) {
272 if (PadStrategy == TargetOpcode::G_ZEXT)
274 else if (PadStrategy == TargetOpcode::G_ANYEXT)
277 assert(PadStrategy == TargetOpcode::G_SEXT);
298 for (
int I = 0;
I != NumParts; ++
I) {
299 bool AllMergePartsArePadding =
true;
302 for (
int J = 0; J != NumSubParts; ++J) {
303 int Idx =
I * NumSubParts + J;
304 if (
Idx >= NumOrigSrc) {
305 SubMerge[J] = PadReg;
309 SubMerge[J] = VRegs[
Idx];
312 AllMergePartsArePadding =
false;
318 if (AllMergePartsArePadding && !AllPadReg) {
319 if (PadStrategy == TargetOpcode::G_ANYEXT)
321 else if (PadStrategy == TargetOpcode::G_ZEXT)
331 Remerge[
I] = AllPadReg;
335 if (NumSubParts == 1)
336 Remerge[
I] = SubMerge[0];
341 if (AllMergePartsArePadding && !AllPadReg)
342 AllPadReg = Remerge[
I];
345 VRegs = std::move(Remerge);
349void LegalizerHelper::buildWidenedRemergeToDst(
Register DstReg,
LLT LCMTy,
356 if (DstTy == LCMTy) {
370 UnmergeDefs[0] = DstReg;
371 for (
unsigned I = 1;
I != NumDefs; ++
I)
383#define RTLIBCASE_INT(LibcallPrefix) \
387 return RTLIB::LibcallPrefix##32; \
389 return RTLIB::LibcallPrefix##64; \
391 return RTLIB::LibcallPrefix##128; \
393 llvm_unreachable("unexpected size"); \
397#define RTLIBCASE(LibcallPrefix) \
401 return RTLIB::LibcallPrefix##32; \
403 return RTLIB::LibcallPrefix##64; \
405 return RTLIB::LibcallPrefix##80; \
407 return RTLIB::LibcallPrefix##128; \
409 llvm_unreachable("unexpected size"); \
414 case TargetOpcode::G_MUL:
416 case TargetOpcode::G_SDIV:
418 case TargetOpcode::G_UDIV:
420 case TargetOpcode::G_SREM:
422 case TargetOpcode::G_UREM:
424 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
426 case TargetOpcode::G_FADD:
428 case TargetOpcode::G_FSUB:
430 case TargetOpcode::G_FMUL:
432 case TargetOpcode::G_FDIV:
434 case TargetOpcode::G_FEXP:
436 case TargetOpcode::G_FEXP2:
438 case TargetOpcode::G_FEXP10:
440 case TargetOpcode::G_FREM:
442 case TargetOpcode::G_FPOW:
444 case TargetOpcode::G_FPOWI:
446 case TargetOpcode::G_FMA:
448 case TargetOpcode::G_FSIN:
450 case TargetOpcode::G_FCOS:
452 case TargetOpcode::G_FLOG10:
454 case TargetOpcode::G_FLOG:
456 case TargetOpcode::G_FLOG2:
458 case TargetOpcode::G_FLDEXP:
460 case TargetOpcode::G_FCEIL:
462 case TargetOpcode::G_FFLOOR:
464 case TargetOpcode::G_FMINNUM:
466 case TargetOpcode::G_FMAXNUM:
468 case TargetOpcode::G_FSQRT:
470 case TargetOpcode::G_FRINT:
472 case TargetOpcode::G_FNEARBYINT:
474 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
476 case TargetOpcode::G_INTRINSIC_LRINT:
478 case TargetOpcode::G_INTRINSIC_LLRINT:
504 if (CallerAttrs.
hasRetAttr(Attribute::ZExt) ||
516 if (
MI.getOpcode() == TargetOpcode::G_BZERO)
523 if (!VReg.
isVirtual() || VReg != Next->getOperand(1).getReg())
526 Register PReg = Next->getOperand(0).getReg();
534 if (Ret->getNumImplicitOperands() != 1)
537 if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg())
561 Info.OrigRet = Result;
564 (Result.Ty->isVoidTy() ||
569 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
570 if (!CLI.lowerCall(MIRBuilder,
Info))
573 if (
MI &&
Info.LoweredTailCall) {
574 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
585 "Expected instr following MI to be return or debug inst?");
589 }
while (
MI->getNextNode());
619 Args.push_back({MO.getReg(), OpType, 0});
621 {
MI.getOperand(0).
getReg(), OpType, 0}, Args,
632 for (
unsigned i = 0; i <
MI.getNumOperands() - 1; ++i) {
636 LLT OpLLT =
MRI.getType(Reg);
637 Type *OpTy =
nullptr;
642 Args.push_back({Reg, OpTy, 0});
648 unsigned Opc =
MI.getOpcode();
650 case TargetOpcode::G_BZERO:
651 RTLibcall = RTLIB::BZERO;
653 case TargetOpcode::G_MEMCPY:
654 RTLibcall = RTLIB::MEMCPY;
655 Args[0].Flags[0].setReturned();
657 case TargetOpcode::G_MEMMOVE:
658 RTLibcall = RTLIB::MEMMOVE;
659 Args[0].Flags[0].setReturned();
661 case TargetOpcode::G_MEMSET:
662 RTLibcall = RTLIB::MEMSET;
663 Args[0].Flags[0].setReturned();
668 const char *
Name = TLI.getLibcallName(RTLibcall);
678 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
682 MI.getOperand(
MI.getNumOperands() - 1).getImm() &&
685 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
686 if (!CLI.lowerCall(MIRBuilder,
Info))
689 if (
Info.LoweredTailCall) {
690 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
701 "Expected instr following MI to be return or debug inst?");
705 }
while (
MI.getNextNode());
715 unsigned Opc =
MI.getOpcode();
716 auto &AtomicMI = cast<GMemOperation>(
MI);
717 auto &MMO = AtomicMI.getMMO();
718 auto Ordering = MMO.getMergedOrdering();
719 LLT MemType = MMO.getMemoryType();
722 return RTLIB::UNKNOWN_LIBCALL;
724#define LCALLS(A, B) \
725 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
727 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
729 case TargetOpcode::G_ATOMIC_CMPXCHG:
730 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
732 return getOutlineAtomicHelper(LC, Ordering, MemSize);
734 case TargetOpcode::G_ATOMICRMW_XCHG: {
736 return getOutlineAtomicHelper(LC, Ordering, MemSize);
738 case TargetOpcode::G_ATOMICRMW_ADD:
739 case TargetOpcode::G_ATOMICRMW_SUB: {
741 return getOutlineAtomicHelper(LC, Ordering, MemSize);
743 case TargetOpcode::G_ATOMICRMW_AND: {
745 return getOutlineAtomicHelper(LC, Ordering, MemSize);
747 case TargetOpcode::G_ATOMICRMW_OR: {
749 return getOutlineAtomicHelper(LC, Ordering, MemSize);
751 case TargetOpcode::G_ATOMICRMW_XOR: {
753 return getOutlineAtomicHelper(LC, Ordering, MemSize);
756 return RTLIB::UNKNOWN_LIBCALL;
769 unsigned Opc =
MI.getOpcode();
771 case TargetOpcode::G_ATOMIC_CMPXCHG:
772 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
775 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
776 MI.getFirst4RegLLTs();
779 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
780 std::tie(Ret, RetLLT,
Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New,
781 NewLLT) =
MI.getFirst5RegLLTs();
791 case TargetOpcode::G_ATOMICRMW_XCHG:
792 case TargetOpcode::G_ATOMICRMW_ADD:
793 case TargetOpcode::G_ATOMICRMW_SUB:
794 case TargetOpcode::G_ATOMICRMW_AND:
795 case TargetOpcode::G_ATOMICRMW_OR:
796 case TargetOpcode::G_ATOMICRMW_XOR: {
797 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] =
MI.getFirst3RegLLTs();
800 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
804 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
819 const char *
Name = TLI.getLibcallName(RTLibcall);
829 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
833 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
834 if (!CLI.lowerCall(MIRBuilder,
Info))
846 case TargetOpcode::G_FPEXT:
848 case TargetOpcode::G_FPTRUNC:
850 case TargetOpcode::G_FPTOSI:
852 case TargetOpcode::G_FPTOUI:
854 case TargetOpcode::G_SITOFP:
856 case TargetOpcode::G_UITOFP:
868 {{
MI.getOperand(1).
getReg(), FromType, 0}}, LocObserver, &
MI);
874 switch (
MI.getOpcode()) {
875 case TargetOpcode::G_GET_FPENV:
876 RTLibcall = RTLIB::FEGETENV;
878 case TargetOpcode::G_SET_FPENV:
879 case TargetOpcode::G_RESET_FPENV:
880 RTLibcall = RTLIB::FESETENV;
882 case TargetOpcode::G_GET_FPMODE:
883 RTLibcall = RTLIB::FEGETMODE;
885 case TargetOpcode::G_SET_FPMODE:
886 case TargetOpcode::G_RESET_FPMODE:
887 RTLibcall = RTLIB::FESETMODE;
916 auto &Ctx = MF.getFunction().getContext();
927 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
934 LocObserver,
nullptr);
956 auto &Ctx = MF.getFunction().getContext();
972 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
978 LocObserver,
nullptr);
995 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
997 unsigned PtrSize =
DL.getPointerSizeInBits(AddrSpace);
1014 switch (
MI.getOpcode()) {
1017 case TargetOpcode::G_MUL:
1018 case TargetOpcode::G_SDIV:
1019 case TargetOpcode::G_UDIV:
1020 case TargetOpcode::G_SREM:
1021 case TargetOpcode::G_UREM:
1022 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
1031 case TargetOpcode::G_FADD:
1032 case TargetOpcode::G_FSUB:
1033 case TargetOpcode::G_FMUL:
1034 case TargetOpcode::G_FDIV:
1035 case TargetOpcode::G_FMA:
1036 case TargetOpcode::G_FPOW:
1037 case TargetOpcode::G_FREM:
1038 case TargetOpcode::G_FCOS:
1039 case TargetOpcode::G_FSIN:
1040 case TargetOpcode::G_FLOG10:
1041 case TargetOpcode::G_FLOG:
1042 case TargetOpcode::G_FLOG2:
1043 case TargetOpcode::G_FLDEXP:
1044 case TargetOpcode::G_FEXP:
1045 case TargetOpcode::G_FEXP2:
1046 case TargetOpcode::G_FEXP10:
1047 case TargetOpcode::G_FCEIL:
1048 case TargetOpcode::G_FFLOOR:
1049 case TargetOpcode::G_FMINNUM:
1050 case TargetOpcode::G_FMAXNUM:
1051 case TargetOpcode::G_FSQRT:
1052 case TargetOpcode::G_FRINT:
1053 case TargetOpcode::G_FNEARBYINT:
1054 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1059 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1067 case TargetOpcode::G_INTRINSIC_LRINT:
1068 case TargetOpcode::G_INTRINSIC_LLRINT: {
1075 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1081 {{
MI.getOperand(1).
getReg(), HLTy, 0}}, LocObserver, &
MI);
1084 MI.eraseFromParent();
1087 case TargetOpcode::G_FPOWI: {
1094 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1098 std::initializer_list<CallLowering::ArgInfo> Args = {
1099 {
MI.getOperand(1).getReg(), HLTy, 0},
1100 {
MI.getOperand(2).getReg(), ITy, 1}};
1103 Args, LocObserver, &
MI);
1108 case TargetOpcode::G_FPEXT:
1109 case TargetOpcode::G_FPTRUNC: {
1112 if (!FromTy || !ToTy)
1120 case TargetOpcode::G_FPTOSI:
1121 case TargetOpcode::G_FPTOUI: {
1125 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
1136 case TargetOpcode::G_SITOFP:
1137 case TargetOpcode::G_UITOFP: {
1141 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
1152 case TargetOpcode::G_ATOMICRMW_XCHG:
1153 case TargetOpcode::G_ATOMICRMW_ADD:
1154 case TargetOpcode::G_ATOMICRMW_SUB:
1155 case TargetOpcode::G_ATOMICRMW_AND:
1156 case TargetOpcode::G_ATOMICRMW_OR:
1157 case TargetOpcode::G_ATOMICRMW_XOR:
1158 case TargetOpcode::G_ATOMIC_CMPXCHG:
1159 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1165 case TargetOpcode::G_BZERO:
1166 case TargetOpcode::G_MEMCPY:
1167 case TargetOpcode::G_MEMMOVE:
1168 case TargetOpcode::G_MEMSET: {
1173 MI.eraseFromParent();
1176 case TargetOpcode::G_GET_FPENV:
1177 case TargetOpcode::G_GET_FPMODE: {
1183 case TargetOpcode::G_SET_FPENV:
1184 case TargetOpcode::G_SET_FPMODE: {
1190 case TargetOpcode::G_RESET_FPENV:
1191 case TargetOpcode::G_RESET_FPMODE: {
1200 MI.eraseFromParent();
1210 switch (
MI.getOpcode()) {
1213 case TargetOpcode::G_IMPLICIT_DEF: {
1223 if (SizeOp0 % NarrowSize != 0) {
1224 LLT ImplicitTy = NarrowTy;
1231 MI.eraseFromParent();
1235 int NumParts = SizeOp0 / NarrowSize;
1238 for (
int i = 0; i < NumParts; ++i)
1245 MI.eraseFromParent();
1248 case TargetOpcode::G_CONSTANT: {
1250 const APInt &Val =
MI.getOperand(1).getCImm()->getValue();
1253 int NumParts = TotalSize / NarrowSize;
1256 for (
int I = 0;
I != NumParts; ++
I) {
1257 unsigned Offset =
I * NarrowSize;
1264 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1266 if (LeftoverBits != 0) {
1270 Val.
lshr(NumParts * NarrowSize).
trunc(LeftoverBits));
1274 insertParts(
MI.getOperand(0).getReg(),
1275 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1277 MI.eraseFromParent();
1280 case TargetOpcode::G_SEXT:
1281 case TargetOpcode::G_ZEXT:
1282 case TargetOpcode::G_ANYEXT:
1284 case TargetOpcode::G_TRUNC: {
1290 LLVM_DEBUG(
dbgs() <<
"Can't narrow trunc to type " << NarrowTy <<
"\n");
1296 MI.eraseFromParent();
1300 case TargetOpcode::G_FREEZE: {
1311 for (
unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1317 MI.eraseFromParent();
1320 case TargetOpcode::G_ADD:
1321 case TargetOpcode::G_SUB:
1322 case TargetOpcode::G_SADDO:
1323 case TargetOpcode::G_SSUBO:
1324 case TargetOpcode::G_SADDE:
1325 case TargetOpcode::G_SSUBE:
1326 case TargetOpcode::G_UADDO:
1327 case TargetOpcode::G_USUBO:
1328 case TargetOpcode::G_UADDE:
1329 case TargetOpcode::G_USUBE:
1331 case TargetOpcode::G_MUL:
1332 case TargetOpcode::G_UMULH:
1334 case TargetOpcode::G_EXTRACT:
1336 case TargetOpcode::G_INSERT:
1338 case TargetOpcode::G_LOAD: {
1339 auto &LoadMI = cast<GLoad>(
MI);
1340 Register DstReg = LoadMI.getDstReg();
1345 if (8 * LoadMI.getMemSize().getValue() != DstTy.
getSizeInBits()) {
1349 LoadMI.eraseFromParent();
1355 case TargetOpcode::G_ZEXTLOAD:
1356 case TargetOpcode::G_SEXTLOAD: {
1357 auto &LoadMI = cast<GExtLoad>(
MI);
1358 Register DstReg = LoadMI.getDstReg();
1359 Register PtrReg = LoadMI.getPointerReg();
1362 auto &MMO = LoadMI.getMMO();
1365 if (MemSize == NarrowSize) {
1367 }
else if (MemSize < NarrowSize) {
1369 }
else if (MemSize > NarrowSize) {
1374 if (isa<GZExtLoad>(LoadMI))
1379 LoadMI.eraseFromParent();
1382 case TargetOpcode::G_STORE: {
1383 auto &StoreMI = cast<GStore>(
MI);
1385 Register SrcReg = StoreMI.getValueReg();
1390 int NumParts = SizeOp0 / NarrowSize;
1392 unsigned LeftoverBits = SrcTy.
getSizeInBits() - HandledSize;
1393 if (SrcTy.
isVector() && LeftoverBits != 0)
1396 if (8 * StoreMI.getMemSize().getValue() != SrcTy.
getSizeInBits()) {
1400 StoreMI.eraseFromParent();
1406 case TargetOpcode::G_SELECT:
1408 case TargetOpcode::G_AND:
1409 case TargetOpcode::G_OR:
1410 case TargetOpcode::G_XOR: {
1422 case TargetOpcode::G_SHL:
1423 case TargetOpcode::G_LSHR:
1424 case TargetOpcode::G_ASHR:
1426 case TargetOpcode::G_CTLZ:
1427 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1428 case TargetOpcode::G_CTTZ:
1429 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1430 case TargetOpcode::G_CTPOP:
1432 switch (
MI.getOpcode()) {
1433 case TargetOpcode::G_CTLZ:
1434 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1436 case TargetOpcode::G_CTTZ:
1437 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1439 case TargetOpcode::G_CTPOP:
1449 case TargetOpcode::G_INTTOPTR:
1457 case TargetOpcode::G_PTRTOINT:
1465 case TargetOpcode::G_PHI: {
1468 if (SizeOp0 % NarrowSize != 0)
1471 unsigned NumParts = SizeOp0 / NarrowSize;
1475 for (
unsigned i = 1; i <
MI.getNumOperands(); i += 2) {
1483 for (
unsigned i = 0; i < NumParts; ++i) {
1487 for (
unsigned j = 1; j <
MI.getNumOperands(); j += 2)
1488 MIB.
addUse(SrcRegs[j / 2][i]).
add(
MI.getOperand(j + 1));
1493 MI.eraseFromParent();
1496 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1497 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1501 int OpIdx =
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1507 case TargetOpcode::G_ICMP: {
1526 if (!
extractParts(
MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused,
1527 RHSPartRegs, RHSLeftoverRegs,
MIRBuilder, MRI))
1540 for (
auto LHSAndRHS :
zip(LHSPartRegs, RHSPartRegs)) {
1541 auto LHS = std::get<0>(LHSAndRHS);
1542 auto RHS = std::get<1>(LHSAndRHS);
1550 for (
auto LHSAndRHS :
zip(LHSLeftoverRegs, RHSLeftoverRegs)) {
1551 auto LHS = std::get<0>(LHSAndRHS);
1552 auto RHS = std::get<1>(LHSAndRHS);
1554 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy,
Xor);
1555 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors,
1556 TargetOpcode::G_ZEXT);
1563 assert(Xors.
size() >= 2 &&
"Should have gotten at least two Xors?");
1565 for (
unsigned I = 2, E = Xors.
size();
I < E; ++
I)
1570 assert(LHSPartRegs.
size() == 2 &&
"Expected exactly 2 LHS part regs?");
1571 assert(RHSPartRegs.
size() == 2 &&
"Expected exactly 2 RHS part regs?");
1583 MI.eraseFromParent();
1586 case TargetOpcode::G_FCMP:
1595 case TargetOpcode::G_SEXT_INREG: {
1599 int64_t SizeInBits =
MI.getOperand(2).getImm();
1609 MO1.
setReg(TruncMIB.getReg(0));
1624 if (SizeOp0 % NarrowSize != 0)
1626 int NumParts = SizeOp0 / NarrowSize;
1634 for (
int i = 0; i < NumParts; ++i) {
1650 for (
int i = 0; i < NumParts; ++i) {
1653 PartialExtensionReg = DstRegs.
back();
1655 assert(PartialExtensionReg &&
1656 "Expected to visit partial extension before full");
1657 if (FullExtensionReg) {
1664 FullExtensionReg = DstRegs.
back();
1669 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1672 PartialExtensionReg = DstRegs.
back();
1679 MI.eraseFromParent();
1682 case TargetOpcode::G_BSWAP:
1683 case TargetOpcode::G_BITREVERSE: {
1684 if (SizeOp0 % NarrowSize != 0)
1689 unsigned NumParts = SizeOp0 / NarrowSize;
1690 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
1693 for (
unsigned i = 0; i < NumParts; ++i) {
1695 {SrcRegs[NumParts - 1 - i]});
1702 MI.eraseFromParent();
1705 case TargetOpcode::G_PTR_ADD:
1706 case TargetOpcode::G_PTRMASK: {
1714 case TargetOpcode::G_FPTOUI:
1715 case TargetOpcode::G_FPTOSI:
1717 case TargetOpcode::G_FPEXT:
1724 case TargetOpcode::G_FLDEXP:
1725 case TargetOpcode::G_STRICT_FLDEXP:
1727 case TargetOpcode::G_VSCALE: {
1738 MI.eraseFromParent();
1766 unsigned OpIdx,
unsigned ExtOpcode) {
1769 MO.
setReg(ExtB.getReg(0));
1776 MO.
setReg(ExtB.getReg(0));
1780 unsigned OpIdx,
unsigned TruncOpcode) {
1789 unsigned OpIdx,
unsigned ExtOpcode) {
1828LegalizerHelper::widenScalarMergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1833 auto [DstReg, DstTy, Src1Reg, Src1Ty] =
MI.getFirst2RegLLTs();
1834 if (DstTy.isVector())
1841 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1843 unsigned NumOps =
MI.getNumOperands();
1844 unsigned NumSrc =
MI.getNumOperands() - 1;
1845 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1847 if (WideSize >= DstSize) {
1851 for (
unsigned I = 2;
I != NumOps; ++
I) {
1852 const unsigned Offset = (
I - 1) * PartSize;
1859 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
1865 ResultReg = NextResult;
1868 if (WideSize > DstSize)
1870 else if (DstTy.isPointer())
1873 MI.eraseFromParent();
1898 const int GCD = std::gcd(SrcSize, WideSize);
1909 if (GCD == SrcSize) {
1913 for (
int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1919 if (
static_cast<int>(Unmerges.
size()) != NumMerge * WideSize) {
1921 for (
int I = Unmerges.
size();
I != NumMerge * WideSize; ++
I)
1925 const int PartsPerGCD = WideSize / GCD;
1929 for (
int I = 0;
I != NumMerge; ++
I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1944 MI.eraseFromParent();
1949LegalizerHelper::widenScalarUnmergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1954 int NumDst =
MI.getNumOperands() - 1;
1955 Register SrcReg =
MI.getOperand(NumDst).getReg();
1960 Register Dst0Reg =
MI.getOperand(0).getReg();
1970 dbgs() <<
"Not casting non-integral address space integer\n");
1991 for (
int I = 1;
I != NumDst; ++
I) {
1997 MI.eraseFromParent();
2008 LLVM_DEBUG(
dbgs() <<
"Widening pointer source types not implemented\n");
2033 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2038 if (PartsPerRemerge == 1) {
2041 for (
int I = 0;
I != NumUnmerge; ++
I) {
2044 for (
int J = 0; J != PartsPerUnmerge; ++J) {
2045 int Idx =
I * PartsPerUnmerge + J;
2047 MIB.addDef(
MI.getOperand(
Idx).getReg());
2054 MIB.addUse(Unmerge.getReg(
I));
2058 for (
int J = 0; J != NumUnmerge; ++J)
2059 extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
2062 for (
int I = 0;
I != NumDst; ++
I) {
2063 for (
int J = 0; J < PartsPerRemerge; ++J) {
2064 const int Idx =
I * PartsPerRemerge + J;
2069 RemergeParts.
clear();
2073 MI.eraseFromParent();
2078LegalizerHelper::widenScalarExtract(
MachineInstr &
MI,
unsigned TypeIdx,
2080 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
2081 unsigned Offset =
MI.getOperand(2).getImm();
2084 if (SrcTy.
isVector() || DstTy.isVector())
2100 if (DstTy.isPointer())
2107 MI.eraseFromParent();
2112 LLT ShiftTy = SrcTy;
2121 MI.eraseFromParent();
2152LegalizerHelper::widenScalarInsert(
MachineInstr &
MI,
unsigned TypeIdx,
2154 if (TypeIdx != 0 || WideTy.
isVector())
2164LegalizerHelper::widenScalarAddSubOverflow(
MachineInstr &
MI,
unsigned TypeIdx,
2168 std::optional<Register> CarryIn;
2169 switch (
MI.getOpcode()) {
2172 case TargetOpcode::G_SADDO:
2173 Opcode = TargetOpcode::G_ADD;
2174 ExtOpcode = TargetOpcode::G_SEXT;
2176 case TargetOpcode::G_SSUBO:
2177 Opcode = TargetOpcode::G_SUB;
2178 ExtOpcode = TargetOpcode::G_SEXT;
2180 case TargetOpcode::G_UADDO:
2181 Opcode = TargetOpcode::G_ADD;
2182 ExtOpcode = TargetOpcode::G_ZEXT;
2184 case TargetOpcode::G_USUBO:
2185 Opcode = TargetOpcode::G_SUB;
2186 ExtOpcode = TargetOpcode::G_ZEXT;
2188 case TargetOpcode::G_SADDE:
2189 Opcode = TargetOpcode::G_UADDE;
2190 ExtOpcode = TargetOpcode::G_SEXT;
2191 CarryIn =
MI.getOperand(4).getReg();
2193 case TargetOpcode::G_SSUBE:
2194 Opcode = TargetOpcode::G_USUBE;
2195 ExtOpcode = TargetOpcode::G_SEXT;
2196 CarryIn =
MI.getOperand(4).getReg();
2198 case TargetOpcode::G_UADDE:
2199 Opcode = TargetOpcode::G_UADDE;
2200 ExtOpcode = TargetOpcode::G_ZEXT;
2201 CarryIn =
MI.getOperand(4).getReg();
2203 case TargetOpcode::G_USUBE:
2204 Opcode = TargetOpcode::G_USUBE;
2205 ExtOpcode = TargetOpcode::G_ZEXT;
2206 CarryIn =
MI.getOperand(4).getReg();
2227 LLT CarryOutTy = MRI.
getType(
MI.getOperand(1).getReg());
2230 {LHSExt, RHSExt, *CarryIn})
2242 MI.eraseFromParent();
2247LegalizerHelper::widenScalarAddSubShlSat(
MachineInstr &
MI,
unsigned TypeIdx,
2249 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2250 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2251 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2252 bool IsShift =
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2253 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2278 {ShiftL, ShiftR},
MI.getFlags());
2286 MI.eraseFromParent();
2291LegalizerHelper::widenScalarMulo(
MachineInstr &
MI,
unsigned TypeIdx,
2300 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULO;
2303 LLT OverflowTy = MRI.
getType(OriginalOverflow);
2310 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2319 WideMulCanOverflow ?
MI.getOpcode() : (
unsigned)TargetOpcode::G_MUL;
2322 if (WideMulCanOverflow)
2324 {LeftOperand, RightOperand});
2345 if (WideMulCanOverflow) {
2353 MI.eraseFromParent();
2359 switch (
MI.getOpcode()) {
2362 case TargetOpcode::G_ATOMICRMW_XCHG:
2363 case TargetOpcode::G_ATOMICRMW_ADD:
2364 case TargetOpcode::G_ATOMICRMW_SUB:
2365 case TargetOpcode::G_ATOMICRMW_AND:
2366 case TargetOpcode::G_ATOMICRMW_OR:
2367 case TargetOpcode::G_ATOMICRMW_XOR:
2368 case TargetOpcode::G_ATOMICRMW_MIN:
2369 case TargetOpcode::G_ATOMICRMW_MAX:
2370 case TargetOpcode::G_ATOMICRMW_UMIN:
2371 case TargetOpcode::G_ATOMICRMW_UMAX:
2372 assert(TypeIdx == 0 &&
"atomicrmw with second scalar type");
2378 case TargetOpcode::G_ATOMIC_CMPXCHG:
2379 assert(TypeIdx == 0 &&
"G_ATOMIC_CMPXCHG with second scalar type");
2386 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2396 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2401 case TargetOpcode::G_EXTRACT:
2402 return widenScalarExtract(
MI, TypeIdx, WideTy);
2403 case TargetOpcode::G_INSERT:
2404 return widenScalarInsert(
MI, TypeIdx, WideTy);
2405 case TargetOpcode::G_MERGE_VALUES:
2406 return widenScalarMergeValues(
MI, TypeIdx, WideTy);
2407 case TargetOpcode::G_UNMERGE_VALUES:
2408 return widenScalarUnmergeValues(
MI, TypeIdx, WideTy);
2409 case TargetOpcode::G_SADDO:
2410 case TargetOpcode::G_SSUBO:
2411 case TargetOpcode::G_UADDO:
2412 case TargetOpcode::G_USUBO:
2413 case TargetOpcode::G_SADDE:
2414 case TargetOpcode::G_SSUBE:
2415 case TargetOpcode::G_UADDE:
2416 case TargetOpcode::G_USUBE:
2417 return widenScalarAddSubOverflow(
MI, TypeIdx, WideTy);
2418 case TargetOpcode::G_UMULO:
2419 case TargetOpcode::G_SMULO:
2420 return widenScalarMulo(
MI, TypeIdx, WideTy);
2421 case TargetOpcode::G_SADDSAT:
2422 case TargetOpcode::G_SSUBSAT:
2423 case TargetOpcode::G_SSHLSAT:
2424 case TargetOpcode::G_UADDSAT:
2425 case TargetOpcode::G_USUBSAT:
2426 case TargetOpcode::G_USHLSAT:
2427 return widenScalarAddSubShlSat(
MI, TypeIdx, WideTy);
2428 case TargetOpcode::G_CTTZ:
2429 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2430 case TargetOpcode::G_CTLZ:
2431 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2432 case TargetOpcode::G_CTPOP: {
2443 unsigned ExtOpc =
MI.getOpcode() == TargetOpcode::G_CTTZ ||
2444 MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
2445 ? TargetOpcode::G_ANYEXT
2446 : TargetOpcode::G_ZEXT;
2449 unsigned NewOpc =
MI.getOpcode();
2450 if (NewOpc == TargetOpcode::G_CTTZ) {
2459 NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF;
2465 if (
MI.getOpcode() == TargetOpcode::G_CTLZ ||
2466 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
2474 MI.eraseFromParent();
2477 case TargetOpcode::G_BSWAP: {
2486 MI.getOperand(0).setReg(DstExt);
2499 case TargetOpcode::G_BITREVERSE: {
2508 MI.getOperand(0).setReg(DstExt);
2517 case TargetOpcode::G_FREEZE:
2524 case TargetOpcode::G_ABS:
2531 case TargetOpcode::G_ADD:
2532 case TargetOpcode::G_AND:
2533 case TargetOpcode::G_MUL:
2534 case TargetOpcode::G_OR:
2535 case TargetOpcode::G_XOR:
2536 case TargetOpcode::G_SUB:
2537 case TargetOpcode::G_SHUFFLE_VECTOR:
2548 case TargetOpcode::G_SBFX:
2549 case TargetOpcode::G_UBFX:
2563 case TargetOpcode::G_SHL:
2579 case TargetOpcode::G_ROTR:
2580 case TargetOpcode::G_ROTL:
2589 case TargetOpcode::G_SDIV:
2590 case TargetOpcode::G_SREM:
2591 case TargetOpcode::G_SMIN:
2592 case TargetOpcode::G_SMAX:
2600 case TargetOpcode::G_SDIVREM:
2609 case TargetOpcode::G_ASHR:
2610 case TargetOpcode::G_LSHR:
2614 unsigned CvtOp =
MI.getOpcode() == TargetOpcode::G_ASHR ?
2615 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2628 case TargetOpcode::G_UDIV:
2629 case TargetOpcode::G_UREM:
2630 case TargetOpcode::G_UMIN:
2631 case TargetOpcode::G_UMAX:
2639 case TargetOpcode::G_UDIVREM:
2648 case TargetOpcode::G_SELECT:
2665 case TargetOpcode::G_FPTOSI:
2666 case TargetOpcode::G_FPTOUI:
2667 case TargetOpcode::G_INTRINSIC_LRINT:
2668 case TargetOpcode::G_INTRINSIC_LLRINT:
2669 case TargetOpcode::G_IS_FPCLASS:
2679 case TargetOpcode::G_SITOFP:
2689 case TargetOpcode::G_UITOFP:
2699 case TargetOpcode::G_LOAD:
2700 case TargetOpcode::G_SEXTLOAD:
2701 case TargetOpcode::G_ZEXTLOAD:
2707 case TargetOpcode::G_STORE: {
2718 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2724 case TargetOpcode::G_CONSTANT: {
2728 MRI.
getType(
MI.getOperand(0).getReg()));
2729 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
2730 ExtOpc == TargetOpcode::G_ANYEXT) &&
2733 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2737 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
2743 case TargetOpcode::G_FCONSTANT: {
2751 MI.eraseFromParent();
2754 case TargetOpcode::G_IMPLICIT_DEF: {
2760 case TargetOpcode::G_BRCOND:
2766 case TargetOpcode::G_FCMP:
2777 case TargetOpcode::G_ICMP:
2783 MI.getOperand(1).getPredicate()))
2784 ? TargetOpcode::G_SEXT
2785 : TargetOpcode::G_ZEXT;
2792 case TargetOpcode::G_PTR_ADD:
2793 assert(TypeIdx == 1 &&
"unable to legalize pointer of G_PTR_ADD");
2799 case TargetOpcode::G_PHI: {
2800 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
2803 for (
unsigned I = 1;
I <
MI.getNumOperands();
I += 2) {
2815 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2823 TargetOpcode::G_ANYEXT);
2838 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2874 case TargetOpcode::G_FADD:
2875 case TargetOpcode::G_FMUL:
2876 case TargetOpcode::G_FSUB:
2877 case TargetOpcode::G_FMA:
2878 case TargetOpcode::G_FMAD:
2879 case TargetOpcode::G_FNEG:
2880 case TargetOpcode::G_FABS:
2881 case TargetOpcode::G_FCANONICALIZE:
2882 case TargetOpcode::G_FMINNUM:
2883 case TargetOpcode::G_FMAXNUM:
2884 case TargetOpcode::G_FMINNUM_IEEE:
2885 case TargetOpcode::G_FMAXNUM_IEEE:
2886 case TargetOpcode::G_FMINIMUM:
2887 case TargetOpcode::G_FMAXIMUM:
2888 case TargetOpcode::G_FDIV:
2889 case TargetOpcode::G_FREM:
2890 case TargetOpcode::G_FCEIL:
2891 case TargetOpcode::G_FFLOOR:
2892 case TargetOpcode::G_FCOS:
2893 case TargetOpcode::G_FSIN:
2894 case TargetOpcode::G_FLOG10:
2895 case TargetOpcode::G_FLOG:
2896 case TargetOpcode::G_FLOG2:
2897 case TargetOpcode::G_FRINT:
2898 case TargetOpcode::G_FNEARBYINT:
2899 case TargetOpcode::G_FSQRT:
2900 case TargetOpcode::G_FEXP:
2901 case TargetOpcode::G_FEXP2:
2902 case TargetOpcode::G_FEXP10:
2903 case TargetOpcode::G_FPOW:
2904 case TargetOpcode::G_INTRINSIC_TRUNC:
2905 case TargetOpcode::G_INTRINSIC_ROUND:
2906 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2910 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2916 case TargetOpcode::G_FPOWI:
2917 case TargetOpcode::G_FLDEXP:
2918 case TargetOpcode::G_STRICT_FLDEXP: {
2920 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP)
2941 case TargetOpcode::G_FFREXP: {
2954 case TargetOpcode::G_INTTOPTR:
2962 case TargetOpcode::G_PTRTOINT:
2970 case TargetOpcode::G_BUILD_VECTOR: {
2974 for (
int I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2988 case TargetOpcode::G_SEXT_INREG:
2997 case TargetOpcode::G_PTRMASK: {
3005 case TargetOpcode::G_VECREDUCE_FADD:
3006 case TargetOpcode::G_VECREDUCE_FMUL:
3007 case TargetOpcode::G_VECREDUCE_FMIN:
3008 case TargetOpcode::G_VECREDUCE_FMAX:
3009 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3010 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3024 case TargetOpcode::G_VSCALE: {
3031 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
3036 case TargetOpcode::G_SPLAT_VECTOR: {
3050 auto Unmerge =
B.buildUnmerge(Ty, Src);
3051 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
3060 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
3062 LLT DstLLT =
MRI.getType(DstReg);
3083 MI.eraseFromParent();
3094 MI.eraseFromParent();
3101 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
3106 if (DstTy.isVector()) {
3107 int NumDstElt = DstTy.getNumElements();
3111 LLT DstCastTy = DstEltTy;
3112 LLT SrcPartTy = SrcEltTy;
3116 if (NumSrcElt < NumDstElt) {
3126 SrcPartTy = SrcEltTy;
3127 }
else if (NumSrcElt > NumDstElt) {
3138 DstCastTy = DstEltTy;
3148 MI.eraseFromParent();
3152 if (DstTy.isVector()) {
3156 MI.eraseFromParent();
3172 unsigned NewEltSize,
3173 unsigned OldEltSize) {
3174 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3175 LLT IdxTy =
B.getMRI()->getType(
Idx);
3178 auto OffsetMask =
B.buildConstant(
3180 auto OffsetIdx =
B.buildAnd(IdxTy,
Idx, OffsetMask);
3181 return B.buildShl(IdxTy, OffsetIdx,
3182 B.buildConstant(IdxTy,
Log2_32(OldEltSize))).getReg(0);
3197 auto [Dst, DstTy, SrcVec, SrcVecTy,
Idx, IdxTy] =
MI.getFirst3RegLLTs();
3201 unsigned OldNumElts = SrcVecTy.getNumElements();
3208 if (NewNumElts > OldNumElts) {
3219 if (NewNumElts % OldNumElts != 0)
3223 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3232 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
3236 NewOps[
I] = Elt.getReg(0);
3241 MI.eraseFromParent();
3245 if (NewNumElts < OldNumElts) {
3246 if (NewEltSize % OldEltSize != 0)
3268 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3287 MI.eraseFromParent();
3301 LLT TargetTy =
B.getMRI()->getType(TargetReg);
3302 LLT InsertTy =
B.getMRI()->getType(InsertReg);
3303 auto ZextVal =
B.buildZExt(TargetTy, InsertReg);
3304 auto ShiftedInsertVal =
B.buildShl(TargetTy, ZextVal, OffsetBits);
3307 auto EltMask =
B.buildConstant(
3311 auto ShiftedMask =
B.buildShl(TargetTy, EltMask, OffsetBits);
3312 auto InvShiftedMask =
B.buildNot(TargetTy, ShiftedMask);
3315 auto MaskedOldElt =
B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
3319 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
3333 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy,
Idx, IdxTy] =
3334 MI.getFirst4RegLLTs();
3346 if (NewNumElts < OldNumElts) {
3347 if (NewEltSize % OldEltSize != 0)
3356 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3376 CastTy, CastVec, InsertedElt, ScaledIdx).
getReg(0);
3380 MI.eraseFromParent();
3399 if (MemSizeInBits != MemStoreSizeInBits) {
3419 if (isa<GSExtLoad>(LoadMI)) {
3422 }
else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) {
3431 if (DstTy != LoadTy)
3457 uint64_t LargeSplitSize, SmallSplitSize;
3462 SmallSplitSize = MemSizeInBits - LargeSplitSize;
3472 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3497 LargeSplitSize / 8);
3501 SmallPtr, *SmallMMO);
3506 if (AnyExtTy == DstTy)
3541 if (StoreWidth != StoreSizeInBits) {
3576 uint64_t LargeSplitSize, SmallSplitSize;
3579 LargeSplitSize = llvm::bit_floor<uint64_t>(MemTy.
getSizeInBits());
3586 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3625 switch (
MI.getOpcode()) {
3626 case TargetOpcode::G_LOAD: {
3641 case TargetOpcode::G_STORE: {
3657 case TargetOpcode::G_SELECT: {
3663 dbgs() <<
"bitcast action not implemented for vector select\n");
3674 case TargetOpcode::G_AND:
3675 case TargetOpcode::G_OR:
3676 case TargetOpcode::G_XOR: {
3684 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3686 case TargetOpcode::G_INSERT_VECTOR_ELT:
3694void LegalizerHelper::changeOpcode(
MachineInstr &
MI,
unsigned NewOpcode) {
3702 using namespace TargetOpcode;
3704 switch(
MI.getOpcode()) {
3707 case TargetOpcode::G_FCONSTANT:
3709 case TargetOpcode::G_BITCAST:
3711 case TargetOpcode::G_SREM:
3712 case TargetOpcode::G_UREM: {
3716 {MI.getOperand(1), MI.getOperand(2)});
3720 MI.eraseFromParent();
3723 case TargetOpcode::G_SADDO:
3724 case TargetOpcode::G_SSUBO:
3726 case TargetOpcode::G_UMULH:
3727 case TargetOpcode::G_SMULH:
3729 case TargetOpcode::G_SMULO:
3730 case TargetOpcode::G_UMULO: {
3733 auto [Res, Overflow,
LHS,
RHS] =
MI.getFirst4Regs();
3736 unsigned Opcode =
MI.getOpcode() == TargetOpcode::G_SMULO
3737 ? TargetOpcode::G_SMULH
3738 : TargetOpcode::G_UMULH;
3742 MI.setDesc(
TII.get(TargetOpcode::G_MUL));
3743 MI.removeOperand(1);
3754 if (Opcode == TargetOpcode::G_SMULH) {
3763 case TargetOpcode::G_FNEG: {
3764 auto [Res, SubByReg] =
MI.getFirst2Regs();
3774 MI.eraseFromParent();
3777 case TargetOpcode::G_FSUB:
3778 case TargetOpcode::G_STRICT_FSUB: {
3779 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
3785 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
3790 MI.eraseFromParent();
3793 case TargetOpcode::G_FMAD:
3795 case TargetOpcode::G_FFLOOR:
3797 case TargetOpcode::G_INTRINSIC_ROUND:
3799 case TargetOpcode::G_FRINT: {
3802 changeOpcode(
MI, TargetOpcode::G_INTRINSIC_ROUNDEVEN);
3805 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
3806 auto [OldValRes, SuccessRes,
Addr, CmpVal, NewVal] =
MI.getFirst5Regs();
3809 **
MI.memoperands_begin());
3812 MI.eraseFromParent();
3815 case TargetOpcode::G_LOAD:
3816 case TargetOpcode::G_SEXTLOAD:
3817 case TargetOpcode::G_ZEXTLOAD:
3819 case TargetOpcode::G_STORE:
3821 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
3822 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
3823 case TargetOpcode::G_CTLZ:
3824 case TargetOpcode::G_CTTZ:
3825 case TargetOpcode::G_CTPOP:
3828 auto [Res, CarryOut,
LHS,
RHS] =
MI.getFirst4Regs();
3837 MI.eraseFromParent();
3841 auto [Res, CarryOut,
LHS,
RHS, CarryIn] =
MI.getFirst5Regs();
3867 MI.eraseFromParent();
3871 auto [Res, BorrowOut,
LHS,
RHS] =
MI.getFirst4Regs();
3876 MI.eraseFromParent();
3880 auto [Res, BorrowOut,
LHS,
RHS, BorrowIn] =
MI.getFirst5Regs();
3902 MI.eraseFromParent();
3927 case G_MERGE_VALUES:
3929 case G_UNMERGE_VALUES:
3931 case TargetOpcode::G_SEXT_INREG: {
3932 assert(
MI.getOperand(2).isImm() &&
"Expected immediate");
3933 int64_t SizeInBits =
MI.getOperand(2).getImm();
3935 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
3942 MI.eraseFromParent();
3945 case G_EXTRACT_VECTOR_ELT:
3946 case G_INSERT_VECTOR_ELT:
3948 case G_SHUFFLE_VECTOR:
3950 case G_DYN_STACKALLOC:
3954 case G_STACKRESTORE:
3964 case G_READ_REGISTER:
3965 case G_WRITE_REGISTER:
4010 case G_MEMCPY_INLINE:
4011 return lowerMemcpyInline(
MI);
4042 unsigned AddrSpace =
DL.getAllocaAddrSpace();
4051 LLT IdxTy =
B.getMRI()->getType(IdxReg);
4063 return B.buildAnd(IdxTy, IdxReg,
B.buildConstant(IdxTy, Imm)).getReg(0);
4066 return B.buildUMin(IdxTy, IdxReg,
B.buildConstant(IdxTy, NElts - 1))
4077 "Converting bits to bytes lost precision");
4084 unsigned IndexSizeInBits =
DL.getIndexSize(AS) * 8;
4101 std::initializer_list<unsigned> NonVecOpIndices) {
4102 if (
MI.getNumMemOperands() != 0)
4105 LLT VecTy =
MRI.getType(
MI.getReg(0));
4110 for (
unsigned OpIdx = 1; OpIdx <
MI.getNumOperands(); ++OpIdx) {
4143 int NumParts, NumLeftover;
4144 std::tie(NumParts, NumLeftover) =
4147 assert(NumParts > 0 &&
"Error in getNarrowTypeBreakDown");
4148 for (
int i = 0; i < NumParts; ++i) {
4153 assert(NumLeftover == 1 &&
"expected exactly one leftover");
4162 for (
unsigned i = 0; i <
N; ++i) {
4165 else if (
Op.isImm())
4167 else if (
Op.isPredicate())
4189 std::initializer_list<unsigned> NonVecOpIndices) {
4191 "Non-compatible opcode or not specified non-vector operands");
4194 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4195 unsigned NumDefs =
MI.getNumDefs();
4203 for (
unsigned i = 0; i < NumDefs; ++i) {
4212 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4213 ++UseIdx, ++UseNo) {
4216 MI.getOperand(UseIdx));
4221 for (
auto Reg : SplitPieces)
4226 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4230 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4232 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4233 Defs.
push_back(OutputOpsPieces[DstNo][i]);
4236 for (
unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
4237 Uses.push_back(InputOpsPieces[InputNo][i]);
4240 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4241 OutputRegs[DstNo].push_back(
I.getReg(DstNo));
4246 for (
unsigned i = 0; i < NumDefs; ++i)
4247 mergeMixedSubvectors(
MI.getReg(i), OutputRegs[i]);
4249 for (
unsigned i = 0; i < NumDefs; ++i)
4253 MI.eraseFromParent();
4262 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4263 unsigned NumDefs =
MI.getNumDefs();
4272 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4273 UseIdx += 2, ++UseNo) {
4281 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4283 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4289 for (
unsigned j = 0; j < NumInputs / 2; ++j) {
4290 Phi.addUse(InputOpsPieces[j][i]);
4291 Phi.add(
MI.getOperand(1 + j * 2 + 1));
4301 mergeMixedSubvectors(
MI.getReg(0), OutputRegs);
4306 MI.eraseFromParent();
4314 const int NumDst =
MI.getNumOperands() - 1;
4315 const Register SrcReg =
MI.getOperand(NumDst).getReg();
4319 if (TypeIdx != 1 || NarrowTy == DstTy)
4345 const int PartsPerUnmerge = NumDst / NumUnmerge;
4347 for (
int I = 0;
I != NumUnmerge; ++
I) {
4350 for (
int J = 0; J != PartsPerUnmerge; ++J)
4351 MIB.
addDef(
MI.getOperand(
I * PartsPerUnmerge + J).getReg());
4352 MIB.
addUse(Unmerge.getReg(
I));
4355 MI.eraseFromParent();
4362 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4366 assert(DstTy.isVector() && NarrowTy.
isVector() &&
"Expected vector types");
4368 if (NarrowTy == SrcTy)
4378 if ((DstTy.getSizeInBits() % NarrowTy.
getSizeInBits() != 0) ||
4392 for (
unsigned i = 1; i <
MI.getNumOperands(); ++i) {
4394 for (
unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
4400 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
4401 for (
unsigned i = 0,
Offset = 0; i < NumNarrowTyPieces;
4402 ++i,
Offset += NumNarrowTyElts) {
4409 MI.eraseFromParent();
4413 assert(TypeIdx == 0 &&
"Bad type index");
4429 unsigned NumParts = DstTy.getNumElements() / NarrowTy.
getNumElements();
4432 for (
unsigned i = 0; i < NumParts; ++i) {
4434 for (
unsigned j = 0; j < NumElts; ++j)
4435 Sources.
push_back(
MI.getOperand(1 + i * NumElts + j).getReg());
4441 MI.eraseFromParent();
4449 auto [DstReg, SrcVec] =
MI.getFirst2Regs();
4451 bool IsInsert =
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
4453 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) &&
"not a vector type index");
4455 InsertVal =
MI.getOperand(2).getReg();
4470 IdxVal = MaybeCst->Value.getSExtValue();
4474 MI.eraseFromParent();
4479 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
4482 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
4483 TargetOpcode::G_ANYEXT);
4488 int64_t PartIdx = IdxVal / NewNumElts;
4497 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
4498 VecParts[PartIdx] = InsertPart.getReg(0);
4502 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
4507 MI.eraseFromParent();
4531 bool IsLoad = isa<GLoad>(LdStMI);
4543 int NumLeftover = -1;
4549 if (
extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
4551 NumParts = NarrowRegs.
size();
4552 NumLeftover = NarrowLeftoverRegs.
size();
4569 auto MMO = LdStMI.
getMMO();
4571 unsigned NumParts,
unsigned Offset) ->
unsigned {
4574 for (
unsigned Idx = 0, E = NumParts;
Idx != E &&
Offset < TotalSize;
4576 unsigned ByteOffset =
Offset / 8;
4586 ValRegs.push_back(Dst);
4598 unsigned HandledOffset =
4599 splitTypePieces(NarrowTy, NarrowRegs, NumParts,
Offset);
4603 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
4606 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
4607 LeftoverTy, NarrowLeftoverRegs);
4617 using namespace TargetOpcode;
4621 switch (
MI.getOpcode()) {
4622 case G_IMPLICIT_DEF:
4638 case G_FCANONICALIZE:
4655 case G_INTRINSIC_ROUND:
4656 case G_INTRINSIC_ROUNDEVEN:
4657 case G_INTRINSIC_TRUNC:
4676 case G_FMINNUM_IEEE:
4677 case G_FMAXNUM_IEEE:
4697 case G_CTLZ_ZERO_UNDEF:
4699 case G_CTTZ_ZERO_UNDEF:
4713 case G_ADDRSPACE_CAST:
4726 case G_STRICT_FLDEXP:
4740 case G_UNMERGE_VALUES:
4742 case G_BUILD_VECTOR:
4743 assert(TypeIdx == 0 &&
"not a vector type index");
4745 case G_CONCAT_VECTORS:
4749 case G_EXTRACT_VECTOR_ELT:
4750 case G_INSERT_VECTOR_ELT:
4759 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
4760 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
4762 case G_SHUFFLE_VECTOR:
4768 case G_INTRINSIC_FPTRUNC_ROUND:
4778 assert(
MI.getOpcode() == TargetOpcode::G_BITCAST &&
4779 "Not a bitcast operation");
4784 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4792 if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy)
4797 for (
unsigned i = 0; i < SrcVRegs.
size(); i++)
4802 MI.eraseFromParent();
4808 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
4812 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
4813 MI.getFirst3RegLLTs();
4816 if (DstTy != Src1Ty)
4818 if (DstTy != Src2Ty)
4833 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
4849 unsigned InputUsed[2] = {-1U, -1U};
4850 unsigned FirstMaskIdx =
High * NewElts;
4851 bool UseBuildVector =
false;
4852 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4854 int Idx = Mask[FirstMaskIdx + MaskOffset];
4859 if (Input >= std::size(Inputs)) {
4866 Idx -= Input * NewElts;
4870 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
4871 if (InputUsed[OpNo] == Input) {
4874 }
else if (InputUsed[OpNo] == -1U) {
4876 InputUsed[OpNo] = Input;
4881 if (OpNo >= std::size(InputUsed)) {
4884 UseBuildVector =
true;
4892 if (UseBuildVector) {
4897 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4899 int Idx = Mask[FirstMaskIdx + MaskOffset];
4904 if (Input >= std::size(Inputs)) {
4911 Idx -= Input * NewElts;
4915 .buildExtractVectorElement(
4916 EltTy, Inputs[Input],
4923 }
else if (InputUsed[0] == -1U) {
4927 Register Op0 = Inputs[InputUsed[0]];
4931 : Inputs[InputUsed[1]];
4940 MI.eraseFromParent();
4946 auto &RdxMI = cast<GVecReduce>(
MI);
4953 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
4959 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
4962 const unsigned NumParts =
4968 if (DstTy != NarrowTy)
4974 unsigned NumPartsLeft = NumParts;
4975 while (NumPartsLeft > 1) {
4976 for (
unsigned Idx = 0;
Idx < NumPartsLeft - 1;
Idx += 2) {
4979 .buildInstr(ScalarOpc, {NarrowTy},
4980 {SplitSrcs[
Idx], SplitSrcs[
Idx + 1]})
4983 SplitSrcs = PartialResults;
4984 PartialResults.
clear();
4985 NumPartsLeft = SplitSrcs.
size();
4989 MI.eraseFromParent();
4994 for (
unsigned Idx = 1;
Idx < NumParts; ++
Idx)
4998 MI.eraseFromParent();
5002 for (
unsigned Part = 0; Part < NumParts; ++Part) {
5012 return tryNarrowPow2Reduction(
MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
5015 Register Acc = PartialReductions[0];
5016 for (
unsigned Part = 1; Part < NumParts; ++Part) {
5017 if (Part == NumParts - 1) {
5019 {Acc, PartialReductions[Part]});
5022 .
buildInstr(ScalarOpc, {DstTy}, {Acc, PartialReductions[Part]})
5026 MI.eraseFromParent();
5032 unsigned int TypeIdx,
5034 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
5035 MI.getFirst3RegLLTs();
5036 if (!NarrowTy.
isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
5040 assert((
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
5041 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
5042 "Unexpected vecreduce opcode");
5043 unsigned ScalarOpc =
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
5044 ? TargetOpcode::G_FADD
5045 : TargetOpcode::G_FMUL;
5051 for (
unsigned i = 0; i < NumParts; i++)
5056 MI.eraseFromParent();
5063 unsigned ScalarOpc) {
5071 while (SplitSrcs.
size() > 1) {
5073 for (
unsigned Idx = 0;
Idx < SplitSrcs.
size()-1;
Idx += 2) {
5081 SplitSrcs = std::move(PartialRdxs);
5085 MI.getOperand(1).setReg(SplitSrcs[0]);
5092 const LLT HalfTy,
const LLT AmtTy) {
5100 MI.eraseFromParent();
5106 unsigned VTBits = 2 * NVTBits;
5109 if (
MI.getOpcode() == TargetOpcode::G_SHL) {
5110 if (Amt.
ugt(VTBits)) {
5112 }
else if (Amt.
ugt(NVTBits)) {
5116 }
else if (Amt == NVTBits) {
5127 }
else if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5128 if (Amt.
ugt(VTBits)) {
5130 }
else if (Amt.
ugt(NVTBits)) {
5134 }
else if (Amt == NVTBits) {
5148 if (Amt.
ugt(VTBits)) {
5151 }
else if (Amt.
ugt(NVTBits)) {
5156 }
else if (Amt == NVTBits) {
5173 MI.eraseFromParent();
5197 if (DstEltSize % 2 != 0)
5203 const unsigned NewBitSize = DstEltSize / 2;
5229 switch (
MI.getOpcode()) {
5230 case TargetOpcode::G_SHL: {
5246 ResultRegs[0] =
Lo.getReg(0);
5247 ResultRegs[1] =
Hi.getReg(0);
5250 case TargetOpcode::G_LSHR:
5251 case TargetOpcode::G_ASHR: {
5261 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5275 ResultRegs[0] =
Lo.getReg(0);
5276 ResultRegs[1] =
Hi.getReg(0);
5284 MI.eraseFromParent();
5291 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
5294 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
5309 assert(Ty.
isScalar() &&
"Expected scalar type to make neutral element for");
5314 "getNeutralElementForVecReduce called with invalid opcode!");
5315 case TargetOpcode::G_VECREDUCE_ADD:
5316 case TargetOpcode::G_VECREDUCE_OR:
5317 case TargetOpcode::G_VECREDUCE_XOR:
5318 case TargetOpcode::G_VECREDUCE_UMAX:
5320 case TargetOpcode::G_VECREDUCE_MUL:
5322 case TargetOpcode::G_VECREDUCE_AND:
5323 case TargetOpcode::G_VECREDUCE_UMIN:
5326 case TargetOpcode::G_VECREDUCE_SMAX:
5329 case TargetOpcode::G_VECREDUCE_SMIN:
5332 case TargetOpcode::G_VECREDUCE_FADD:
5334 case TargetOpcode::G_VECREDUCE_FMUL:
5336 case TargetOpcode::G_VECREDUCE_FMINIMUM:
5337 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
5338 assert(
false &&
"getNeutralElementForVecReduce unimplemented for "
5339 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
5347 unsigned Opc =
MI.getOpcode();
5349 case TargetOpcode::G_IMPLICIT_DEF:
5350 case TargetOpcode::G_LOAD: {
5358 case TargetOpcode::G_STORE:
5365 case TargetOpcode::G_AND:
5366 case TargetOpcode::G_OR:
5367 case TargetOpcode::G_XOR:
5368 case TargetOpcode::G_ADD:
5369 case TargetOpcode::G_SUB:
5370 case TargetOpcode::G_MUL:
5371 case TargetOpcode::G_FADD:
5372 case TargetOpcode::G_FSUB:
5373 case TargetOpcode::G_FMUL:
5374 case TargetOpcode::G_FDIV:
5375 case TargetOpcode::G_FCOPYSIGN:
5376 case TargetOpcode::G_UADDSAT:
5377 case TargetOpcode::G_USUBSAT:
5378 case TargetOpcode::G_SADDSAT:
5379 case TargetOpcode::G_SSUBSAT:
5380 case TargetOpcode::G_SMIN:
5381 case TargetOpcode::G_SMAX:
5382 case TargetOpcode::G_UMIN:
5383 case TargetOpcode::G_UMAX:
5384 case TargetOpcode::G_FMINNUM:
5385 case TargetOpcode::G_FMAXNUM:
5386 case TargetOpcode::G_FMINNUM_IEEE:
5387 case TargetOpcode::G_FMAXNUM_IEEE:
5388 case TargetOpcode::G_FMINIMUM:
5389 case TargetOpcode::G_FMAXIMUM:
5390 case TargetOpcode::G_STRICT_FADD:
5391 case TargetOpcode::G_STRICT_FSUB:
5392 case TargetOpcode::G_STRICT_FMUL:
5393 case TargetOpcode::G_SHL:
5394 case TargetOpcode::G_ASHR:
5395 case TargetOpcode::G_LSHR: {
5403 case TargetOpcode::G_FMA:
5404 case TargetOpcode::G_STRICT_FMA:
5405 case TargetOpcode::G_FSHR:
5406 case TargetOpcode::G_FSHL: {
5415 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
5416 case TargetOpcode::G_EXTRACT:
5423 case TargetOpcode::G_INSERT:
5424 case TargetOpcode::G_INSERT_VECTOR_ELT:
5425 case TargetOpcode::G_FREEZE:
5426 case TargetOpcode::G_FNEG:
5427 case TargetOpcode::G_FABS:
5428 case TargetOpcode::G_FSQRT:
5429 case TargetOpcode::G_FCEIL:
5430 case TargetOpcode::G_FFLOOR:
5431 case TargetOpcode::G_FNEARBYINT:
5432 case TargetOpcode::G_FRINT:
5433 case TargetOpcode::G_INTRINSIC_ROUND:
5434 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
5435 case TargetOpcode::G_INTRINSIC_TRUNC:
5436 case TargetOpcode::G_BSWAP:
5437 case TargetOpcode::G_FCANONICALIZE:
5438 case TargetOpcode::G_SEXT_INREG:
5439 case TargetOpcode::G_ABS:
5447 case TargetOpcode::G_SELECT: {
5448 auto [DstReg, DstTy, CondReg, CondTy] =
MI.getFirst2RegLLTs();
5450 if (!CondTy.isScalar() ||
5458 MI.getOperand(1).setReg(ShufSplat.getReg(0));
5463 if (CondTy.isVector())
5473 case TargetOpcode::G_UNMERGE_VALUES:
5475 case TargetOpcode::G_PHI:
5477 case TargetOpcode::G_SHUFFLE_VECTOR:
5479 case TargetOpcode::G_BUILD_VECTOR: {
5481 for (
auto Op :
MI.uses()) {
5491 MI.eraseFromParent();
5494 case TargetOpcode::G_SEXT:
5495 case TargetOpcode::G_ZEXT:
5496 case TargetOpcode::G_ANYEXT:
5497 case TargetOpcode::G_TRUNC:
5498 case TargetOpcode::G_FPTRUNC:
5499 case TargetOpcode::G_FPEXT:
5500 case TargetOpcode::G_FPTOSI:
5501 case TargetOpcode::G_FPTOUI:
5502 case TargetOpcode::G_SITOFP:
5503 case TargetOpcode::G_UITOFP: {
5523 case TargetOpcode::G_ICMP:
5524 case TargetOpcode::G_FCMP: {
5538 case TargetOpcode::G_BITCAST: {
5559 case TargetOpcode::G_VECREDUCE_FADD:
5560 case TargetOpcode::G_VECREDUCE_FMUL:
5561 case TargetOpcode::G_VECREDUCE_ADD:
5562 case TargetOpcode::G_VECREDUCE_MUL:
5563 case TargetOpcode::G_VECREDUCE_AND:
5564 case TargetOpcode::G_VECREDUCE_OR:
5565 case TargetOpcode::G_VECREDUCE_XOR:
5566 case TargetOpcode::G_VECREDUCE_SMAX:
5567 case TargetOpcode::G_VECREDUCE_SMIN:
5568 case TargetOpcode::G_VECREDUCE_UMAX:
5569 case TargetOpcode::G_VECREDUCE_UMIN: {
5573 auto NeutralElement = getNeutralElementForVecReduce(
5581 NeutralElement,
Idx);
5585 MO.
setReg(NewVec.getReg(0));
5597 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
5599 unsigned MaskNumElts = Mask.size();
5603 if (MaskNumElts == SrcNumElts)
5606 if (MaskNumElts < SrcNumElts) {
5610 for (
unsigned I = MaskNumElts;
I < SrcNumElts; ++
I)
5616 MI.getOperand(1).getReg(),
5617 MI.getOperand(2).getReg(), NewMask);
5618 MI.eraseFromParent();
5623 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
5624 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
5632 MOps1[0] =
MI.getOperand(1).getReg();
5633 MOps2[0] =
MI.getOperand(2).getReg();
5640 for (
unsigned I = 0;
I != MaskNumElts; ++
I) {
5642 if (
Idx >=
static_cast<int>(SrcNumElts))
5643 Idx += PaddedMaskNumElts - SrcNumElts;
5648 if (MaskNumElts != PaddedMaskNumElts) {
5653 for (
unsigned I = 0;
I < MaskNumElts; ++
I) {
5663 MI.eraseFromParent();
5669 unsigned int TypeIdx,
LLT MoreTy) {
5670 auto [DstTy, Src1Ty, Src2Ty] =
MI.getFirst3LLTs();
5672 unsigned NumElts = DstTy.getNumElements();
5675 if (DstTy.isVector() && Src1Ty.isVector() &&
5676 DstTy.getNumElements() != Src1Ty.getNumElements()) {
5684 if (DstTy != Src1Ty || DstTy != Src2Ty)
5692 for (
unsigned I = 0;
I != NumElts; ++
I) {
5694 if (
Idx <
static_cast<int>(NumElts))
5699 for (
unsigned I = NumElts;
I != WidenNumElts; ++
I)
5704 MI.getOperand(1).getReg(),
5705 MI.getOperand(2).getReg(), NewMask);
5706 MI.eraseFromParent();
5715 unsigned SrcParts = Src1Regs.
size();
5716 unsigned DstParts = DstRegs.
size();
5718 unsigned DstIdx = 0;
5720 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
5721 DstRegs[DstIdx] = FactorSum;
5723 unsigned CarrySumPrevDstIdx;
5726 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
5728 for (
unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
5729 i <= std::min(DstIdx, SrcParts - 1); ++i) {
5731 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
5735 for (
unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
5736 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
5738 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
5748 if (DstIdx != DstParts - 1) {
5750 B.buildUAddo(NarrowTy,
LLT::scalar(1), Factors[0], Factors[1]);
5751 FactorSum = Uaddo.
getReg(0);
5752 CarrySum =
B.buildZExt(NarrowTy, Uaddo.
getReg(1)).
getReg(0);
5753 for (
unsigned i = 2; i < Factors.
size(); ++i) {
5755 B.buildUAddo(NarrowTy,
LLT::scalar(1), FactorSum, Factors[i]);
5756 FactorSum = Uaddo.
getReg(0);
5758 CarrySum =
B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
5762 FactorSum =
B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
5763 for (
unsigned i = 2; i < Factors.
size(); ++i)
5764 FactorSum =
B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
5767 CarrySumPrevDstIdx = CarrySum;
5768 DstRegs[DstIdx] = FactorSum;
5785 unsigned Opcode =
MI.getOpcode();
5786 unsigned OpO, OpE, OpF;
5788 case TargetOpcode::G_SADDO:
5789 case TargetOpcode::G_SADDE:
5790 case TargetOpcode::G_UADDO:
5791 case TargetOpcode::G_UADDE:
5792 case TargetOpcode::G_ADD:
5793 OpO = TargetOpcode::G_UADDO;
5794 OpE = TargetOpcode::G_UADDE;
5795 OpF = TargetOpcode::G_UADDE;
5796 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
5797 OpF = TargetOpcode::G_SADDE;
5799 case TargetOpcode::G_SSUBO:
5800 case TargetOpcode::G_SSUBE:
5801 case TargetOpcode::G_USUBO:
5802 case TargetOpcode::G_USUBE:
5803 case TargetOpcode::G_SUB:
5804 OpO = TargetOpcode::G_USUBO;
5805 OpE = TargetOpcode::G_USUBE;
5806 OpF = TargetOpcode::G_USUBE;
5807 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
5808 OpF = TargetOpcode::G_SSUBE;
5815 unsigned NumDefs =
MI.getNumExplicitDefs();
5816 Register Src1 =
MI.getOperand(NumDefs).getReg();
5817 Register Src2 =
MI.getOperand(NumDefs + 1).getReg();
5820 CarryDst =
MI.getOperand(1).getReg();
5821 if (
MI.getNumOperands() == NumDefs + 3)
5822 CarryIn =
MI.getOperand(NumDefs + 2).getReg();
5825 LLT LeftoverTy, DummyTy;
5827 extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left,
5832 int NarrowParts = Src1Regs.
size();
5833 for (
int I = 0, E = Src1Left.
size();
I != E; ++
I) {
5839 for (
int i = 0, e = Src1Regs.
size(); i != e; ++i) {
5844 if (i == e - 1 && CarryDst)
5845 CarryOut = CarryDst;
5849 {Src1Regs[i], Src2Regs[i]});
5850 }
else if (i == e - 1) {
5852 {Src1Regs[i], Src2Regs[i], CarryIn});
5855 {Src1Regs[i], Src2Regs[i], CarryIn});
5861 insertParts(
MI.getOperand(0).getReg(), RegTy, NarrowTy,
5862 ArrayRef(DstRegs).take_front(NarrowParts), LeftoverTy,
5863 ArrayRef(DstRegs).drop_front(NarrowParts));
5865 MI.eraseFromParent();
5871 auto [DstReg, Src1, Src2] =
MI.getFirst3Regs();
5879 if (
Size % NarrowSize != 0)
5882 unsigned NumParts =
Size / NarrowSize;
5883 bool IsMulHigh =
MI.getOpcode() == TargetOpcode::G_UMULH;
5884 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
5890 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
5895 MI.eraseFromParent();
5905 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_FPTOSI;
5919 IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
5935 if (SizeOp1 % NarrowSize != 0)
5937 int NumParts = SizeOp1 / NarrowSize;
5941 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
5945 uint64_t OpStart =
MI.getOperand(2).getImm();
5947 for (
int i = 0; i < NumParts; ++i) {
5948 unsigned SrcStart = i * NarrowSize;
5950 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
5953 }
else if (SrcStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
5961 int64_t ExtractOffset;
5963 if (OpStart < SrcStart) {
5965 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
5967 ExtractOffset = OpStart - SrcStart;
5968 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
5972 if (ExtractOffset != 0 || SegSize != NarrowSize) {
5984 else if (DstRegs.
size() > 1)
5988 MI.eraseFromParent();
6003 extractParts(
MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs,
6011 uint64_t OpStart =
MI.getOperand(3).getImm();
6013 for (
int I = 0, E = SrcRegs.
size();
I != E; ++
I) {
6014 unsigned DstStart =
I * NarrowSize;
6016 if (DstStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
6024 if (MRI.
getType(SrcRegs[
I]) == LeftoverTy) {
6030 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
6038 int64_t ExtractOffset, InsertOffset;
6040 if (OpStart < DstStart) {
6042 ExtractOffset = DstStart - OpStart;
6043 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
6045 InsertOffset = OpStart - DstStart;
6048 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
6052 if (ExtractOffset != 0 || SegSize != OpSize) {
6072 MI.eraseFromParent();
6082 assert(
MI.getNumOperands() == 3 && TypeIdx == 0);
6088 if (!
extractParts(
MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
6089 Src0Regs, Src0LeftoverRegs,
MIRBuilder, MRI))
6093 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
6094 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6097 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6099 {Src0Regs[I], Src1Regs[I]});
6103 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6106 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
6107 DstLeftoverRegs.
push_back(Inst.getReg(0));
6110 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6111 LeftoverTy, DstLeftoverRegs);
6113 MI.eraseFromParent();
6123 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
6130 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
6131 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
MI.getOpcode());
6132 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
6134 MI.eraseFromParent();
6144 Register CondReg =
MI.getOperand(1).getReg();
6156 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
6157 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6161 if (!
extractParts(
MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
6162 Src2Regs, Src2LeftoverRegs,
MIRBuilder, MRI))
6165 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6167 CondReg, Src1Regs[
I], Src2Regs[
I]);
6171 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6173 LeftoverTy, CondReg, Src1LeftoverRegs[
I], Src2LeftoverRegs[
I]);
6177 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6178 LeftoverTy, DstLeftoverRegs);
6180 MI.eraseFromParent();
6190 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6194 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
6197 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6199 auto C_0 =
B.buildConstant(NarrowTy, 0);
6201 UnmergeSrc.getReg(1), C_0);
6202 auto LoCTLZ = IsUndef ?
6203 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
6204 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
6205 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6206 auto HiIsZeroCTLZ =
B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
6207 auto HiCTLZ =
B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
6208 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
6210 MI.eraseFromParent();
6223 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6227 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
6230 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6232 auto C_0 =
B.buildConstant(NarrowTy, 0);
6234 UnmergeSrc.getReg(0), C_0);
6235 auto HiCTTZ = IsUndef ?
6236 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
6237 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
6238 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6239 auto LoIsZeroCTTZ =
B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
6240 auto LoCTTZ =
B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
6241 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
6243 MI.eraseFromParent();
6256 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6266 MI.eraseFromParent();
6286 auto MinExp =
B.buildConstant(ExpTy,
minIntN(ClampSize));
6287 auto ClampMin =
B.buildSMax(ExpTy, ExpReg, MinExp);
6288 auto MaxExp =
B.buildConstant(ExpTy,
maxIntN(ClampSize));
6289 auto Clamp =
B.buildSMin(ExpTy, ClampMin, MaxExp);
6291 auto Trunc =
B.buildTrunc(NarrowTy, Clamp);
6293 MI.getOperand(2).setReg(Trunc.getReg(0));
6300 unsigned Opc =
MI.getOpcode();
6309 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
6312 MI.setDesc(
TII.get(TargetOpcode::G_CTLZ));
6316 case TargetOpcode::G_CTLZ: {
6317 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6320 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6328 MI.eraseFromParent();
6344 for (
unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
6348 Op = MIBOp.getReg(0);
6353 MI.eraseFromParent();
6356 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
6359 MI.setDesc(
TII.get(TargetOpcode::G_CTTZ));
6363 case TargetOpcode::G_CTTZ: {
6364 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6367 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6376 MI.eraseFromParent();
6387 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
6388 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
6392 MI.eraseFromParent();
6396 MI.setDesc(
TII.get(TargetOpcode::G_CTPOP));
6397 MI.getOperand(1).setReg(MIBTmp.getReg(0));
6401 case TargetOpcode::G_CTPOP: {
6412 auto C_1 =
B.buildConstant(Ty, 1);
6413 auto B2Set1LoTo1Hi =
B.buildLShr(Ty, SrcReg, C_1);
6415 auto C_B2Mask1HiTo0 =
B.buildConstant(Ty, B2Mask1HiTo0);
6416 auto B2Count1Hi =
B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
6417 auto B2Count =
B.buildSub(Ty, SrcReg, B2Count1Hi);
6421 auto C_2 =
B.buildConstant(Ty, 2);
6422 auto B4Set2LoTo2Hi =
B.buildLShr(Ty, B2Count, C_2);
6424 auto C_B4Mask2HiTo0 =
B.buildConstant(Ty, B4Mask2HiTo0);
6425 auto B4HiB2Count =
B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
6426 auto B4LoB2Count =
B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
6427 auto B4Count =
B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
6434 auto C_4 =
B.buildConstant(Ty, 4);
6435 auto B8HiB4Count =
B.buildLShr(Ty, B4Count, C_4);
6436 auto B8CountDirty4Hi =
B.buildAdd(Ty, B8HiB4Count, B4Count);
6438 auto C_B8Mask4HiTo0 =
B.buildConstant(Ty, B8Mask4HiTo0);
6439 auto B8Count =
B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
6441 assert(
Size<=128 &&
"Scalar size is too large for CTPOP lower algorithm");
6447 auto C_SizeM8 =
B.buildConstant(Ty,
Size - 8);
6449 auto IsMulSupported = [
this](
const LLT Ty) {
6450 auto Action = LI.
getAction({TargetOpcode::G_MUL, {Ty}}).Action;
6453 if (IsMulSupported(Ty)) {
6454 auto ResTmp =
B.buildMul(Ty, B8Count, MulMask);
6455 B.buildLShr(
MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
6457 auto ResTmp = B8Count;
6458 for (
unsigned Shift = 8; Shift <
Size; Shift *= 2) {
6459 auto ShiftC =
B.buildConstant(Ty, Shift);
6460 auto Shl =
B.buildShl(Ty, ResTmp, ShiftC);
6461 ResTmp =
B.buildAdd(Ty, ResTmp, Shl);
6463 B.buildLShr(
MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
6465 MI.eraseFromParent();
6478 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(
C);
6486 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6495 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6496 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6519 MI.eraseFromParent();
6525 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6530 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6574 MI.eraseFromParent();
6588 bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6589 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6593 return lowerFunnelShiftAsShifts(
MI);
6597 if (Result == UnableToLegalize)
6598 return lowerFunnelShiftAsShifts(
MI);
6603 auto [Dst, Src] =
MI.getFirst2Regs();
6617 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
6631 {UnmergeSrc.getReg(0)});
6633 {UnmergeSrc.getReg(1)});
6638 MI.eraseFromParent();
6655 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC);
6659 LLT DstTy =
MRI.getType(DstReg);
6660 LLT SrcTy =
MRI.getType(SrcReg);
6680 for (
unsigned I = 0;
I < SplitSrcs.
size(); ++
I) {
6694 MI.eraseFromParent();
6703 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6705 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6706 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6709 MI.eraseFromParent();
6714 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6716 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
6717 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6722 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6725 return lowerRotateWithReverseRotate(
MI);
6728 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6729 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6730 bool IsFShLegal =
false;
6731 if ((IsFShLegal = LI.isLegalOrCustom({FShOpc, {DstTy, AmtTy}})) ||
6732 LI.isLegalOrCustom({RevFsh, {DstTy, AmtTy}})) {
6736 MI.eraseFromParent();
6741 return buildFunnelShift(FShOpc, Dst, Src, Amt);
6744 return buildFunnelShift(RevFsh, Dst, Src, Amt);
6749 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
6750 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
6751 auto BitWidthMinusOneC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits - 1);
6757 auto NegAmt = MIRBuilder.
buildSub(AmtTy, Zero, Amt);
6758 auto ShAmt = MIRBuilder.
buildAnd(AmtTy, Amt, BitWidthMinusOneC);
6760 auto RevAmt = MIRBuilder.
buildAnd(AmtTy, NegAmt, BitWidthMinusOneC);
6766 auto BitWidthC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits);
6767 auto ShAmt = MIRBuilder.
buildURem(AmtTy, Amt, BitWidthC);
6769 auto RevAmt = MIRBuilder.
buildSub(AmtTy, BitWidthMinusOneC, ShAmt);
6771 auto Inner = MIRBuilder.
buildInstr(RevShiftOpc, {DstTy}, {Src, One});
6775 MIRBuilder.
buildOr(Dst, ShVal, RevShiftVal);
6776 MI.eraseFromParent();
6784 auto [Dst, Src] =
MI.getFirst2Regs();
6834 MI.eraseFromParent();
6839 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6845 MI.eraseFromParent();
6864 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6874 MI.eraseFromParent();
6899 MI.eraseFromParent();
6907 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6911 if (SrcTy !=
S64 && SrcTy !=
S32)
6913 if (DstTy !=
S32 && DstTy !=
S64)
6942 MI.eraseFromParent();
6947 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
7006 MI.eraseFromParent();
7016 auto [Dst, Src] =
MI.getFirst2Regs();
7024 unsigned Flags =
MI.getFlags();
7027 MI.eraseFromParent();
7031 const unsigned ExpMask = 0x7ff;
7032 const unsigned ExpBiasf64 = 1023;
7033 const unsigned ExpBiasf16 = 15;
7122 MI.eraseFromParent();
7128 auto [DstTy, SrcTy] =
MI.getFirst2LLTs();
7141 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7146 MI.eraseFromParent();
7152 case TargetOpcode::G_SMIN:
7154 case TargetOpcode::G_SMAX:
7156 case TargetOpcode::G_UMIN:
7158 case TargetOpcode::G_UMAX:
7166 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7174 MI.eraseFromParent();
7180 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] =
MI.getFirst3RegLLTs();
7181 const int Src0Size = Src0Ty.getScalarSizeInBits();
7182 const int Src1Size = Src1Ty.getScalarSizeInBits();
7192 if (Src0Ty == Src1Ty) {
7194 }
else if (Src0Size > Src1Size) {
7209 unsigned Flags =
MI.getFlags();
7212 MI.eraseFromParent();
7218 unsigned NewOp =
MI.getOpcode() == TargetOpcode::G_FMINNUM ?
7219 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
7221 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7241 MI.eraseFromParent();
7249 unsigned Flags =
MI.getFlags();
7254 MI.eraseFromParent();
7260 auto [DstReg,
X] =
MI.getFirst2Regs();
7261 const unsigned Flags =
MI.getFlags();
7288 MI.eraseFromParent();
7293 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
7294 unsigned Flags =
MI.getFlags();
7306 SrcReg, Zero, Flags);
7308 SrcReg, Trunc, Flags);
7313 MI.eraseFromParent();
7319 const unsigned NumOps =
MI.getNumOperands();
7320 auto [DstReg, DstTy, Src0Reg, Src0Ty] =
MI.getFirst2RegLLTs();
7321 unsigned PartSize = Src0Ty.getSizeInBits();
7326 for (
unsigned I = 2;
I != NumOps; ++
I) {
7327 const unsigned Offset = (
I - 1) * PartSize;
7332 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
7338 ResultReg = NextResult;
7341 if (DstTy.isPointer()) {
7343 DstTy.getAddressSpace())) {
7351 MI.eraseFromParent();
7357 const unsigned NumDst =
MI.getNumOperands() - 1;
7358 Register SrcReg =
MI.getOperand(NumDst).getReg();
7359 Register Dst0Reg =
MI.getOperand(0).getReg();
7374 unsigned Offset = DstSize;
7375 for (
unsigned I = 1;
I != NumDst; ++
I,
Offset += DstSize) {
7381 MI.eraseFromParent();
7400 if (
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
7401 InsertVal =
MI.getOperand(2).getReg();
7415 SrcRegs[IdxVal] =
MI.getOperand(2).getReg();
7421 MI.eraseFromParent();
7426 LLVM_DEBUG(
dbgs() <<
"Can't handle non-byte element vectors yet\n");
7444 int64_t
Offset = IdxVal * EltBytes;
7463 MI.eraseFromParent();
7469 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
7470 MI.getFirst3RegLLTs();
7478 for (
int Idx : Mask) {
7480 if (!Undef.isValid())
7486 if (Src0Ty.isScalar()) {
7489 int NumElts = Src0Ty.getNumElements();
7490 Register SrcVec =
Idx < NumElts ? Src0Reg : Src1Reg;
7491 int ExtractIdx =
Idx < NumElts ?
Idx :
Idx - NumElts;
7498 if (DstTy.isScalar())
7502 MI.eraseFromParent();
7519 if (Alignment >
Align(1)) {
7531 const auto &MF = *
MI.getMF();
7532 const auto &TFI = *MF.getSubtarget().getFrameLowering();
7537 Register AllocSize =
MI.getOperand(1).getReg();
7548 MI.eraseFromParent();
7559 MI.eraseFromParent();
7570 MI.eraseFromParent();
7576 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
7577 unsigned Offset =
MI.getOperand(2).getImm();
7582 unsigned DstSize = DstTy.getSizeInBits();
7584 if ((
Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
7591 for (
unsigned Idx =
Offset / SrcEltSize;
7595 if (SubVectorElts.
size() == 1)
7600 MI.eraseFromParent();
7605 if (DstTy.isScalar() &&
7608 LLT SrcIntTy = SrcTy;
7622 MI.eraseFromParent();
7630 auto [Dst, Src, InsertSrc] =
MI.getFirst3Regs();
7642 if ((
Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
7655 for (
unsigned i = 0;
Idx < (
Offset + InsertSize) / EltSize;
7657 DstElts.
push_back(UnmergeInsertSrc.getReg(i));
7670 MI.eraseFromParent();
7684 LLVM_DEBUG(
dbgs() <<
"Not casting non-integral address space integer\n");
7688 LLT IntDstTy = DstTy;
7714 MI.eraseFromParent();
7720 auto [Dst0, Dst0Ty, Dst1, Dst1Ty,
LHS, LHSTy,
RHS, RHSTy] =
7721 MI.getFirst4RegLLTs();
7722 const bool IsAdd =
MI.getOpcode() == TargetOpcode::G_SADDO;
7725 LLT BoolTy = Dst1Ty;
7744 auto ResultLowerThanLHS =
7752 MI.eraseFromParent();
7759 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7764 switch (
MI.getOpcode()) {
7767 case TargetOpcode::G_UADDSAT:
7770 BaseOp = TargetOpcode::G_ADD;
7772 case TargetOpcode::G_SADDSAT:
7775 BaseOp = TargetOpcode::G_ADD;
7777 case TargetOpcode::G_USUBSAT:
7780 BaseOp = TargetOpcode::G_SUB;
7782 case TargetOpcode::G_SSUBSAT:
7785 BaseOp = TargetOpcode::G_SUB;
7828 MI.eraseFromParent();
7834 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7839 unsigned OverflowOp;
7840 switch (
MI.getOpcode()) {
7843 case TargetOpcode::G_UADDSAT:
7846 OverflowOp = TargetOpcode::G_UADDO;
7848 case TargetOpcode::G_SADDSAT:
7851 OverflowOp = TargetOpcode::G_SADDO;
7853 case TargetOpcode::G_USUBSAT:
7856 OverflowOp = TargetOpcode::G_USUBO;
7858 case TargetOpcode::G_SSUBSAT:
7861 OverflowOp = TargetOpcode::G_SSUBO;
7867 Register Tmp = OverflowRes.getReg(0);
7868 Register Ov = OverflowRes.getReg(1);
7894 MI.eraseFromParent();
7900 assert((
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
7901 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
7902 "Expected shlsat opcode!");
7903 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SSHLSAT;
7904 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7926 MI.eraseFromParent();
7931 auto [Dst, Src] =
MI.getFirst2Regs();
7934 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
7943 for (
unsigned i = 1; i < SizeInBytes / 2; ++i) {
7945 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
7957 Res.getInstr()->getOperand(0).setReg(Dst);
7959 MI.eraseFromParent();
7966 const LLT Ty = Dst.getLLTTy(*
B.getMRI());
7969 auto LHS =
B.buildLShr(Ty,
B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
7970 auto RHS =
B.buildAnd(Ty,
B.buildShl(Ty, Src, C_N), MaskLoNTo0);
7971 return B.buildOr(Dst,
LHS,
RHS);
7976 auto [Dst, Src] =
MI.getFirst2Regs();
8000 MI.eraseFromParent();
8008 bool IsRead =
MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
8009 int NameOpIdx = IsRead ? 1 : 0;
8010 int ValRegIndex = IsRead ? 0 : 1;
8012 Register ValReg =
MI.getOperand(ValRegIndex).getReg();
8014 const MDString *RegStr = cast<MDString>(
8015 cast<MDNode>(
MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
8026 MI.eraseFromParent();
8032 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULH;
8033 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
8042 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
8048 MI.eraseFromParent();
8054 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
8059 MI.eraseFromParent();
8064 MI.eraseFromParent();
8083 APInt ExpMask = Inf;
8101 LLT DstTyCopy = DstTy;
8116 Mask &= ~fcPosFinite;
8123 Mask &= ~fcNegFinite;
8134 Mask &= ~PartialCheck;
8143 else if (PartialCheck ==
fcZero)
8162 appendToRes(SubnormalRes);
8169 else if (PartialCheck ==
fcInf)
8182 if (PartialCheck ==
fcNan) {
8186 }
else if (PartialCheck ==
fcQNan) {
8196 Abs, InfWithQnanBitC);
8204 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
8207 APInt MaxExpMinusOne = ExpMask - ExpLSB;
8218 appendToRes(NormalRes);
8222 MI.eraseFromParent();
8228 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
8229 MI.getFirst4RegLLTs();
8231 bool IsEltPtr = DstTy.isPointerOrPointerVector();
8240 if (MaskTy.isScalar()) {
8254 if (DstTy.isVector()) {
8257 MaskReg = ShufSplat.
getReg(0);
8262 }
else if (!DstTy.isVector()) {
8267 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
8280 MI.eraseFromParent();
8286 unsigned Opcode =
MI.getOpcode();
8289 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
8290 : TargetOpcode::G_UDIV,
8291 {
MI.getOperand(0).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8293 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
8294 : TargetOpcode::G_UREM,
8295 {
MI.getOperand(1).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8296 MI.eraseFromParent();
8313 MI.eraseFromParent();
8328 MI.eraseFromParent();
8335 Register DestReg =
MI.getOperand(0).getReg();
8341 MI.eraseFromParent();
8370 Register ListPtr =
MI.getOperand(1).getReg();
8380 const Align A(
MI.getOperand(2).getImm());
8387 VAList = AndDst.
getReg(0);
8405 Align EltAlignment =
DL.getABITypeAlign(Ty);
8410 MI.eraseFromParent();
8425 unsigned Limit,
const MemOp &
Op,
8426 unsigned DstAS,
unsigned SrcAS,
8429 if (
Op.isMemcpyWithFixedDstAlign() &&
Op.getSrcAlign() <
Op.getDstAlign())
8439 if (
Op.isFixedDstAlign())
8447 unsigned NumMemOps = 0;
8451 while (TySize >
Size) {
8460 assert(NewTySize > 0 &&
"Could not find appropriate type");
8467 if (NumMemOps &&
Op.allowOverlap() && NewTySize <
Size &&
8469 VT, DstAS,
Op.isFixedDstAlign() ?
Op.getDstAlign() :
Align(1),
8479 if (++NumMemOps > Limit)
8482 MemOps.push_back(Ty);
8501 if (!Ty.
isVector() && ValVRegAndVal) {
8502 APInt Scalar = ValVRegAndVal->Value.trunc(8);
8510 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
8533 auto &MF = *
MI.getParent()->getParent();
8534 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8535 auto &
DL = MF.getDataLayout();
8538 assert(KnownLen != 0 &&
"Have a zero length memset length!");
8540 bool DstAlignCanChange =
false;
8546 DstAlignCanChange =
true;
8549 std::vector<LLT> MemOps;
8551 const auto &DstMMO = **
MI.memoperands_begin();
8555 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
8563 MF.getFunction().getAttributes(), TLI))
8566 if (DstAlignCanChange) {
8569 Align NewAlign =
DL.getABITypeAlign(IRTy);
8570 if (NewAlign > Alignment) {
8571 Alignment = NewAlign;
8581 LLT LargestTy = MemOps[0];
8582 for (
unsigned i = 1; i < MemOps.size(); i++)
8584 LargestTy = MemOps[i];
8597 unsigned DstOff = 0;
8598 unsigned Size = KnownLen;
8599 for (
unsigned I = 0;
I < MemOps.size();
I++) {
8602 if (TySize >
Size) {
8605 assert(
I == MemOps.size() - 1 &&
I != 0);
8606 DstOff -= TySize -
Size;
8617 Value = MIB.buildTrunc(Ty, MemSetValue).getReg(0);
8624 auto *StoreMMO = MF.getMachineMemOperand(&DstMMO, DstOff, Ty);
8630 Ptr = MIB.buildPtrAdd(PtrTy, Dst,
Offset).getReg(0);
8633 MIB.buildStore(
Value,
Ptr, *StoreMMO);
8638 MI.eraseFromParent();
8644 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8646 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8648 const auto *MMOIt =
MI.memoperands_begin();
8650 bool IsVolatile =
MemOp->isVolatile();
8656 "inline memcpy with dynamic size is not yet supported");
8657 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8658 if (KnownLen == 0) {
8659 MI.eraseFromParent();
8663 const auto &DstMMO = **
MI.memoperands_begin();
8664 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8665 Align DstAlign = DstMMO.getBaseAlign();
8666 Align SrcAlign = SrcMMO.getBaseAlign();
8668 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8675 Align SrcAlign,
bool IsVolatile) {
8676 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8677 return lowerMemcpy(
MI, Dst, Src, KnownLen,
8678 std::numeric_limits<uint64_t>::max(), DstAlign, SrcAlign,
8685 Align SrcAlign,
bool IsVolatile) {
8686 auto &MF = *
MI.getParent()->getParent();
8687 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8688 auto &
DL = MF.getDataLayout();
8691 assert(KnownLen != 0 &&
"Have a zero length memcpy length!");
8693 bool DstAlignCanChange =
false;
8695 Align Alignment = std::min(DstAlign, SrcAlign);
8699 DstAlignCanChange =
true;
8705 std::vector<LLT> MemOps;
8707 const auto &DstMMO = **
MI.memoperands_begin();
8708 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8714 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8717 MF.getFunction().getAttributes(), TLI))
8720 if (DstAlignCanChange) {
8723 Align NewAlign =
DL.getABITypeAlign(IRTy);
8728 if (!
TRI->hasStackRealignment(MF))
8729 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8732 if (NewAlign > Alignment) {
8733 Alignment = NewAlign;
8741 LLVM_DEBUG(
dbgs() <<
"Inlining memcpy: " <<
MI <<
" into loads & stores\n");
8749 unsigned CurrOffset = 0;
8750 unsigned Size = KnownLen;
8751 for (
auto CopyTy : MemOps) {
8754 if (CopyTy.getSizeInBytes() >
Size)
8755 CurrOffset -= CopyTy.getSizeInBytes() -
Size;
8759 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8761 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.getSizeInBytes());
8766 if (CurrOffset != 0) {
8770 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8772 auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO);
8776 if (CurrOffset != 0) {
8778 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8780 MIB.buildStore(LdVal, StorePtr, *StoreMMO);
8781 CurrOffset += CopyTy.getSizeInBytes();
8782 Size -= CopyTy.getSizeInBytes();
8785 MI.eraseFromParent();
8793 auto &MF = *
MI.getParent()->getParent();
8794 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8795 auto &
DL = MF.getDataLayout();
8798 assert(KnownLen != 0 &&
"Have a zero length memmove length!");
8800 bool DstAlignCanChange =
false;
8803 Align Alignment = std::min(DstAlign, SrcAlign);
8807 DstAlignCanChange =
true;
8810 std::vector<LLT> MemOps;
8812 const auto &DstMMO = **
MI.memoperands_begin();
8813 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8822 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8825 MF.getFunction().getAttributes(), TLI))
8828 if (DstAlignCanChange) {
8831 Align NewAlign =
DL.getABITypeAlign(IRTy);
8836 if (!
TRI->hasStackRealignment(MF))
8837 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8840 if (NewAlign > Alignment) {
8841 Alignment = NewAlign;
8849 LLVM_DEBUG(
dbgs() <<
"Inlining memmove: " <<
MI <<
" into loads & stores\n");
8855 unsigned CurrOffset = 0;
8857 for (
auto CopyTy : MemOps) {
8860 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8864 if (CurrOffset != 0) {
8868 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8870 LoadVals.
push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0));
8871 CurrOffset += CopyTy.getSizeInBytes();
8875 for (
unsigned I = 0;
I < MemOps.size(); ++
I) {
8876 LLT CopyTy = MemOps[
I];
8879 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.
getSizeInBytes());
8882 if (CurrOffset != 0) {
8886 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8888 MIB.buildStore(LoadVals[
I], StorePtr, *StoreMMO);
8891 MI.eraseFromParent();
8897 const unsigned Opc =
MI.getOpcode();
8900 assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMMOVE ||
8901 Opc == TargetOpcode::G_MEMSET) &&
8902 "Expected memcpy like instruction");
8904 auto MMOIt =
MI.memoperands_begin();
8909 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8911 if (Opc != TargetOpcode::G_MEMSET) {
8912 assert(MMOIt !=
MI.memoperands_end() &&
"Expected a second MMO on MI");
8914 SrcAlign =
MemOp->getBaseAlign();
8921 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8923 if (KnownLen == 0) {
8924 MI.eraseFromParent();
8928 bool IsVolatile =
MemOp->isVolatile();
8929 if (Opc == TargetOpcode::G_MEMCPY_INLINE)
8930 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8937 if (MaxLen && KnownLen > MaxLen)
8940 if (Opc == TargetOpcode::G_MEMCPY) {
8941 auto &MF = *
MI.getParent()->getParent();
8942 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8945 return lowerMemcpy(
MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign,
8948 if (Opc == TargetOpcode::G_MEMMOVE)
8949 return lowerMemmove(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
8950 if (Opc == TargetOpcode::G_MEMSET)
8951 return lowerMemset(
MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file describes how to lower LLVM calls to machine code calls.
#define GISEL_VECREDUCE_CASES_NONSEQ
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
#define RTLIBCASE_INT(LibcallPrefix)
static bool findGISelOptimalMemOpLowering(std::vector< LLT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI)
static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI)
static Register buildBitFieldInsert(MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits)
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits ...
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size)
static void broadcastSrcOp(SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op)
Operand Op is used on N sub-instructions.
static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI)
True if an instruction is in tail position in its caller.
static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType, LostDebugLocObserver &LocObserver)
static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize)
Figure out the bit offset into a register when coercing a vector index for the wide element type.
static void makeDstOps(SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts)
Fill DstOps with DstOps that have same number of elements combined as the Ty.
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask)
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
static void getUnmergePieces(SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty)
static CmpInst::Predicate minMaxToCompare(unsigned Opc)
static LegalizerHelper::LegalizeResult createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI)
static RTLIB::Libcall getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI)
static std::pair< int, int > getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
Try to break down OrigTy into NarrowTy sized pieces.
static bool hasSameNumEltsOnAllVectorOperands(GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices)
Check that all vector operands have same number of elements.
static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType, LostDebugLocObserver &LocObserver)
static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg, LLT VecTy)
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType)
static void getUnmergeResults(SmallVectorImpl< Register > &Regs, const MachineInstr &MI)
Append the result registers of G_UNMERGE_VALUES MI to Regs.
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
#define RTLIBCASE(LibcallPrefix)
static Type * getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty)
Interface for Targets to specify which operations they can successfully select and how the others sho...
Tracks DebugLocs between checkpoints and verifies that they are transferred.
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
void negate()
Negate this APInt in place.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit, unsigned hiBit)
Wrap version of getBitsSet.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttributes() const
Return true if the builder has IR-level attributes.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ ICMP_ULT
unsigned less than
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool isNonIntegralAddressSpace(unsigned AddrSpace) const
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
Register getValueReg() const
Get the stored value register.
A base class for all GenericMachineInstrs.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isTailCall(const MachineInstr &MI) const override
bool isEquality() const
Return true if this predicate is either EQ or NE.
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isScalable() const
Returns true if the LLT is a scalable vector.
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
This is an important class for using LLVM in a threaded context.
LegalizeResult lowerShlSat(MachineInstr &MI)
LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI)
LegalizeResult equalizeVectorShuffleLengths(MachineInstr &MI)
Equalize source and destination vector sizes of G_SHUFFLE_VECTOR.
LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_INSERT_VECTOR_ELT.
LegalizeResult lowerSITOFP(MachineInstr &MI)
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty)
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LegalizeResult lowerIntrinsicRound(MachineInstr &MI)
void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode)
Legalize a single operand OpIdx of the machine instruction MI as a Use by extending the operand's typ...
LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerSMULH_UMULH(MachineInstr &MI)
LegalizeResult lowerLoad(GAnyLoad &MI)
LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerAbsToAddXor(MachineInstr &MI)
void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
LegalizeResult lowerFConstant(MachineInstr &MI)
LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerBitreverse(MachineInstr &MI)
LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Legalize a vector instruction by increasing the number of vector elements involved and ignoring the a...
LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI)
LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI)
LegalizeResult lowerEXT(MachineInstr &MI)
LegalizeResult lowerStore(GStore &MI)
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI)
MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by truncating the operand's ty...
LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, unsigned NumElts)
LegalizeResult lowerFPTOUI(MachineInstr &MI)
LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize an instruction by reducing the width of the underlying scalar type.
LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerUnmergeValues(MachineInstr &MI)
LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
LegalizeResult lowerBitcast(MachineInstr &MI)
LegalizeResult lowerMinMax(MachineInstr &MI)
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
LegalizeResult lowerInsert(MachineInstr &MI)
LegalizeResult lowerReadWriteRegister(MachineInstr &MI)
LegalizeResult lowerExtract(MachineInstr &MI)
LegalizeResult fewerElementsBitcast(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LLT HalfTy, LLT ShiftAmtTy)
LegalizeResult lowerISFPCLASS(MachineInstr &MI)
LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI)
LegalizeResult lowerFPOWI(MachineInstr &MI)
LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVectorReduction(MachineInstr &MI)
LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult fewerElementsVectorMultiEltType(GenericMachineInstr &MI, unsigned NumElts, std::initializer_list< unsigned > NonVecOpIndices={})
Handles most opcodes.
LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVAArg(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ AlreadyLegal
Instruction was already legal and no change was made to the MachineFunction.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI)
LegalizeResult lowerFCopySign(MachineInstr &MI)
LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &B)
LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI)
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target.
LegalizeResult lowerFunnelShift(MachineInstr &MI)
LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a vector instruction by splitting into multiple components, each acting on the same scalar t...
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LegalizeResult lowerFPTRUNC(MachineInstr &MI)
LegalizeResult lowerFMad(MachineInstr &MI)
LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI)
LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFFloor(MachineInstr &MI)
LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
LegalizeResult lowerFPTOSI(MachineInstr &MI)
LegalizeResult lowerUITOFP(MachineInstr &MI)
LegalizeResult lowerShuffleVector(MachineInstr &MI)
LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerMergeValues(MachineInstr &MI)
LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by producing a vector with und...
LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT.
LegalizeResult lowerRotate(MachineInstr &MI)
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register coerceToScalar(Register Val)
Cast the given value to an LLT::scalar with an equivalent size.
LegalizeResult lowerDIVREM(MachineInstr &MI)
LegalizeResult lowerSelect(MachineInstr &MI)
LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a use by inserting a G_BITCAST to Ca...
void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, unsigned ExtOpcode)
LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Legalize an instruction by emiting a runtime library call instead.
LegalizeResult lowerStackRestore(MachineInstr &MI)
LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerStackSave(MachineInstr &MI)
LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeResult lowerTRUNC(MachineInstr &MI)
LegalizeResult lowerBswap(MachineInstr &MI)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
LegalizeResult lowerConstant(MachineInstr &MI)
void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeResult legalizeInstrStep(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Replace MI by a sequence of legal instructions that can implement the same operation.
virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const
Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while widening a constant of type Small...
bool isLegalOrCustom(const LegalityQuery &Query) const
virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const
Called for instructions with the Custom LegalizationAction.
virtual bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
void checkpoint(bool CheckDebugLocs=true)
Call this to indicate that it's a good point to assess whether locations have been lost.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
StringRef getString() const
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
iterator getFirstTerminatorForward()
Finds the first terminator in a block by scanning forward.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FSUB Op0, Op1.
MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOSI Src0.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx)
Build and insert Res = G_CONSTANT_POOL Idx.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FABS Op0.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildZExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and inserts Res = G_AND Op, LowBitsSet(ImmOp) Since there is no G_ZEXT_INREG like G_SEXT_INREG,...
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFPow(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FPOW Src0, Src1.
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
MachineInstrBuilder buildIntrinsicTrunc(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_INTRINSIC_TRUNC Src0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildShuffleSplat(const DstOp &Res, const SrcOp &Src)
Build and insert a vector splat of a scalar Src using a G_INSERT_VECTOR_ELT and G_SHUFFLE_VECTOR idio...
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ_ZERO_UNDEF Op0, Src0.
MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts)
Build and insert Res = G_VSCALE MinElts.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
unsigned getBoolExtOp(bool IsVec, bool IsFP) const
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert integer negation Zero = G_CONSTANT 0 Res = G_SUB Zero, Op0.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildStrictFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_STRICT_FADD Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTTZ_ZERO_UNDEF Op0, Src0.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_UITOFP Src0.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_SITOFP Src0.
MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a, b, .....
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildInsert(const DstOp &Res, const SrcOp &Src, const SrcOp &Op, unsigned Index)
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_FCOPYSIGN Op0, Op1.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineInstrBuilder buildDeleteTrailingVectorElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x, y, z = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a,...
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, MMO.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool isDebugInstr() const
unsigned getNumOperands() const
Retuns the total number of operands.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
void setType(LLT NewTy)
Reset the tracked memory type.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
const ConstantInt * getCImm() const
void setReg(Register Reg)
Change the register this operand corresponds to.
void setCImm(const ConstantInt *CI)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &) const
LLT returning variant.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
const Triple & getTargetTriple() const
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const CallLowering * getCallLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getDoubleTy(LLVMContext &C)
static Type * getX86_FP80Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static Type * getFP128Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
@ Libcall
The operation should be implemented as a call to some kind of runtime support library.
@ WidenScalar
The operation should be implemented in terms of a wider scalar base-type.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ NarrowScalar
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type.
@ Custom
The target wants to do something special with this combination of operand and type.
@ MoreElements
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, MachineInstr &MI, LostDebugLocObserver &LocObserver)
Create a libcall to memcpy et al.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_READNONE LLT getLCMType(LLT OrigTy, LLT TargetTy)
Return the least common multiple type of OrigTy and TargetTy, by changing the number of vector elemen...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LegalizerHelper::LegalizeResult createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, const CallLowering::ArgInfo &Result, ArrayRef< CallLowering::ArgInfo > Args, CallingConv::ID CC, LostDebugLocObserver &LocObserver, MachineInstr *MI=nullptr)
Helper function that creates a libcall to the given Name using the given calling convention CC.
void extractParts(Register Reg, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Helper function to split a wide generic register into bitwise blocks with the given Type (which impli...
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI)
Returns true if Val can be assumed to never be a signaling NaN.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_READNONE LLT getGCDType(LLT OrigTy, LLT TargetTy)
Return a type where the total size is the greatest common divisor of OrigTy and TargetTy.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
void extractVectorParts(Register Reg, unsigned NumElts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Version which handles irregular sub-vector splits.
static const fltSemantics & IEEEsingle() LLVM_READNONE
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)