58 if (
F.getFnAttribute(
"disable-tail-calls").getValueAsBool())
64 AttrBuilder CallerAttrs(
F.getContext(),
F.getAttributes().getRetAttrs());
65 for (
const auto &Attr :
66 {Attribute::Alignment, Attribute::Dereferenceable,
67 Attribute::DereferenceableOrNull, Attribute::NoAlias,
68 Attribute::NonNull, Attribute::NoUndef, Attribute::Range})
75 if (CallerAttrs.
contains(Attribute::ZExt) ||
76 CallerAttrs.
contains(Attribute::SExt))
87 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
103 Register ArgReg = cast<RegisterSDNode>(
Value->getOperand(1))->getReg();
104 if (
MRI.getLiveInPhysReg(ArgReg) != Reg)
114 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
115 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
116 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
117 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
118 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
119 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
120 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
121 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
122 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
123 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
124 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
125 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
126 Alignment = Call->getParamStackAlign(ArgIdx);
129 "multiple ABI attributes?");
145std::pair<SDValue, SDValue>
155 Args.reserve(Ops.
size());
158 for (
unsigned i = 0; i < Ops.
size(); ++i) {
161 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
164 Entry.IsZExt = !Entry.IsSExt;
168 Entry.IsSExt = Entry.IsZExt =
false;
170 Args.push_back(Entry);
173 if (LC == RTLIB::UNKNOWN_LIBCALL)
181 bool zeroExtend = !signExtend;
185 signExtend = zeroExtend =
false;
196 return LowerCallTo(CLI);
200 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &
Op,
unsigned DstAS,
202 if (Limit != ~
unsigned(0) &&
Op.isMemcpyWithFixedDstAlign() &&
203 Op.getSrcAlign() <
Op.getDstAlign())
208 if (VT == MVT::Other) {
213 if (
Op.isFixedDstAlign())
231 unsigned NumMemOps = 0;
235 while (VTSize >
Size) {
246 else if (NewVT == MVT::i64 &&
258 if (NewVT == MVT::i8)
267 if (NumMemOps &&
Op.allowOverlap() && NewVTSize <
Size &&
269 VT, DstAS,
Op.isFixedDstAlign() ?
Op.getDstAlign() :
Align(1),
279 if (++NumMemOps > Limit)
282 MemOps.push_back(VT);
297 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
307 bool IsSignaling)
const {
312 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
313 &&
"Unsupported setcc type!");
316 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
317 bool ShouldInvertCC =
false;
321 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
322 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
323 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
327 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
328 (VT == MVT::f64) ? RTLIB::UNE_F64 :
329 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
333 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
334 (VT == MVT::f64) ? RTLIB::OGE_F64 :
335 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
339 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
340 (VT == MVT::f64) ? RTLIB::OLT_F64 :
341 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
345 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
346 (VT == MVT::f64) ? RTLIB::OLE_F64 :
347 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
351 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
352 (VT == MVT::f64) ? RTLIB::OGT_F64 :
353 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
356 ShouldInvertCC =
true;
359 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
360 (VT == MVT::f64) ? RTLIB::UO_F64 :
361 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
365 ShouldInvertCC =
true;
368 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
369 (VT == MVT::f64) ? RTLIB::UO_F64 :
370 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
371 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
372 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
373 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
377 ShouldInvertCC =
true;
380 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
381 (VT == MVT::f64) ? RTLIB::OGE_F64 :
382 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
385 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
386 (VT == MVT::f64) ? RTLIB::OGT_F64 :
387 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
390 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
391 (VT == MVT::f64) ? RTLIB::OLE_F64 :
392 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
395 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
396 (VT == MVT::f64) ? RTLIB::OLT_F64 :
397 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
405 SDValue Ops[2] = {NewLHS, NewRHS};
410 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
415 if (ShouldInvertCC) {
417 CCCode = getSetCCInverse(CCCode, RetVT);
420 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
427 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
430 CCCode = getSetCCInverse(CCCode, RetVT);
431 NewLHS = DAG.
getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
445 if (!isPositionIndependent())
459 unsigned JTEncoding = getJumpTableEncoding();
495 if (!
TM.shouldAssumeDSOLocal(GV))
499 if (isPositionIndependent())
515 const APInt &DemandedElts,
518 unsigned Opcode =
Op.getOpcode();
526 if (targetShrinkDemandedConstant(
Op,
DemandedBits, DemandedElts, TLO))
536 auto *Op1C = dyn_cast<ConstantSDNode>(
Op.getOperand(1));
537 if (!Op1C || Op1C->isOpaque())
541 const APInt &
C = Op1C->getAPIntValue();
546 EVT VT =
Op.getValueType();
563 EVT VT =
Op.getValueType();
578 "ShrinkDemandedOp only supports binary operators!");
579 assert(
Op.getNode()->getNumValues() == 1 &&
580 "ShrinkDemandedOp only supports nodes with one result!");
582 EVT VT =
Op.getValueType();
592 if (!
Op.getNode()->hasOneUse())
605 Op.getOpcode(), dl, SmallVT,
608 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
623 bool Simplified = SimplifyDemandedBits(
Op,
DemandedBits, Known, TLO);
632 const APInt &DemandedElts,
652 bool AssumeSingleUse)
const {
653 EVT VT =
Op.getValueType();
669 EVT VT =
Op.getValueType();
687 switch (
Op.getOpcode()) {
693 EVT SrcVT = Src.getValueType();
694 EVT DstVT =
Op.getValueType();
700 if (NumSrcEltBits == NumDstEltBits)
701 if (
SDValue V = SimplifyMultipleUseDemandedBits(
705 if (SrcVT.
isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
706 unsigned Scale = NumDstEltBits / NumSrcEltBits;
710 for (
unsigned i = 0; i != Scale; ++i) {
711 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
712 unsigned BitOffset = EltOffset * NumSrcEltBits;
715 DemandedSrcBits |= Sub;
716 for (
unsigned j = 0; j != NumElts; ++j)
718 DemandedSrcElts.
setBit((j * Scale) + i);
722 if (
SDValue V = SimplifyMultipleUseDemandedBits(
723 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
728 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
729 unsigned Scale = NumSrcEltBits / NumDstEltBits;
733 for (
unsigned i = 0; i != NumElts; ++i)
734 if (DemandedElts[i]) {
735 unsigned Offset = (i % Scale) * NumDstEltBits;
737 DemandedSrcElts.
setBit(i / Scale);
740 if (
SDValue V = SimplifyMultipleUseDemandedBits(
741 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
762 return Op.getOperand(0);
764 return Op.getOperand(1);
775 return Op.getOperand(0);
777 return Op.getOperand(1);
787 return Op.getOperand(0);
789 return Op.getOperand(1);
795 if (
const APInt *MaxSA =
798 unsigned ShAmt = MaxSA->getZExtValue();
799 unsigned NumSignBits =
802 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
831 EVT ExVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
838 if (NumSignBits >= (
BitWidth - ExBits + 1))
851 EVT SrcVT = Src.getValueType();
852 EVT DstVT =
Op.getValueType();
853 if (IsLE && DemandedElts == 1 &&
866 auto *CIdx = dyn_cast<ConstantSDNode>(
Op.getOperand(2));
869 !DemandedElts[CIdx->getZExtValue()])
883 if (DemandedSubElts == 0)
893 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
894 for (
unsigned i = 0; i != NumElts; ++i) {
895 int M = ShuffleMask[i];
896 if (M < 0 || !DemandedElts[i])
899 IdentityLHS &= (M == (int)i);
900 IdentityRHS &= ((M - NumElts) == i);
906 return Op.getOperand(0);
908 return Op.getOperand(1);
918 if (
SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
928 unsigned Depth)
const {
929 EVT VT =
Op.getValueType();
936 return SimplifyMultipleUseDemandedBits(
Op,
DemandedBits, DemandedElts, DAG,
942 unsigned Depth)
const {
944 return SimplifyMultipleUseDemandedBits(
Op,
DemandedBits, DemandedElts, DAG,
953 const APInt &DemandedElts,
956 "SRL or SRA node is required here!");
959 if (!N1C || !N1C->
isOne())
1005 unsigned ShiftOpc =
Op.getOpcode();
1006 bool IsSigned =
false;
1010 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
1015 unsigned NumZero = std::min(NumZeroA, NumZeroB);
1021 if (NumZero >= 2 && NumSigned < NumZero) {
1026 if (NumSigned >= 1) {
1034 if (NumZero >= 1 && NumSigned < NumZero) {
1054 EVT VT =
Op.getValueType();
1066 Add.getOperand(1)) &&
1091 unsigned Depth,
bool AssumeSingleUse)
const {
1094 "Mask size mismatches value type size!");
1099 EVT VT =
Op.getValueType();
1101 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
1103 "Unexpected vector size");
1106 APInt DemandedElts = OriginalDemandedElts;
1126 cast<ConstantFPSDNode>(
Op)->getValueAPF().bitcastToAPInt());
1131 bool HasMultiUse =
false;
1132 if (!AssumeSingleUse && !
Op.getNode()->hasOneUse()) {
1141 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1150 switch (
Op.getOpcode()) {
1154 if (!DemandedElts[0])
1159 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1161 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO,
Depth + 1))
1166 if (DemandedElts == 1)
1179 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO,
Depth + 1))
1188 auto *LD = cast<LoadSDNode>(
Op);
1189 if (getTargetConstantFromLoad(LD)) {
1195 EVT MemVT = LD->getMemoryVT();
1207 auto *CIdx = dyn_cast<ConstantSDNode>(
Op.getOperand(2));
1212 APInt DemandedVecElts(DemandedElts);
1214 unsigned Idx = CIdx->getZExtValue();
1218 if (!DemandedElts[
Idx])
1225 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO,
Depth + 1))
1231 if (SimplifyDemandedBits(Vec,
DemandedBits, DemandedVecElts, KnownVec, TLO,
1235 if (!!DemandedVecElts)
1250 APInt DemandedSrcElts = DemandedElts;
1254 if (SimplifyDemandedBits(Sub,
DemandedBits, DemandedSubElts, KnownSub, TLO,
1257 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1263 if (!!DemandedSubElts)
1265 if (!!DemandedSrcElts)
1271 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1273 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1275 if (NewSub || NewSrc) {
1276 NewSub = NewSub ? NewSub : Sub;
1277 NewSrc = NewSrc ? NewSrc : Src;
1290 if (Src.getValueType().isScalableVector())
1293 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1296 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, Known, TLO,
1302 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1317 EVT SubVT =
Op.getOperand(0).getValueType();
1320 for (
unsigned i = 0; i != NumSubVecs; ++i) {
1321 APInt DemandedSubElts =
1322 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
1323 if (SimplifyDemandedBits(
Op.getOperand(i),
DemandedBits, DemandedSubElts,
1324 Known2, TLO,
Depth + 1))
1327 if (!!DemandedSubElts)
1337 APInt DemandedLHS, DemandedRHS;
1342 if (!!DemandedLHS || !!DemandedRHS) {
1348 if (!!DemandedLHS) {
1349 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedLHS, Known2, TLO,
1354 if (!!DemandedRHS) {
1355 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedRHS, Known2, TLO,
1362 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1364 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1366 if (DemandedOp0 || DemandedOp1) {
1367 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1368 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1403 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1415 unsigned NumSubElts =
1432 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1437 Known2, TLO,
Depth + 1))
1460 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1462 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1464 if (DemandedOp0 || DemandedOp1) {
1465 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1466 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1479 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1481 if (Flags.hasDisjoint()) {
1482 Flags.setDisjoint(
false);
1483 Op->setFlags(Flags);
1488 if (SimplifyDemandedBits(Op0, ~Known.
One &
DemandedBits, DemandedElts,
1489 Known2, TLO,
Depth + 1)) {
1490 if (Flags.hasDisjoint()) {
1491 Flags.setDisjoint(
false);
1492 Op->setFlags(Flags);
1513 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1515 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1517 if (DemandedOp0 || DemandedOp1) {
1518 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1519 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1530 for (
int I = 0;
I != 2; ++
I) {
1533 SDValue Alt =
Op.getOperand(1 -
I).getOperand(0);
1534 SDValue C2 =
Op.getOperand(1 -
I).getOperand(1);
1536 for (
int J = 0; J != 2; ++J) {
1559 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1563 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedElts, Known2, TLO,
1591 if (
C->getAPIntValue() == Known2.
One) {
1600 if (!
C->isAllOnes() &&
DemandedBits.isSubsetOf(
C->getAPIntValue())) {
1612 if (ShiftC->getAPIntValue().ult(
BitWidth)) {
1613 uint64_t ShiftAmt = ShiftC->getZExtValue();
1616 : Ones.
lshr(ShiftAmt);
1634 if (!
C || !
C->isAllOnes())
1640 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1642 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1644 if (DemandedOp0 || DemandedOp1) {
1645 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1646 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1656 if (SimplifyDemandedBits(
Op.getOperand(2),
DemandedBits, DemandedElts,
1657 Known, TLO,
Depth + 1))
1659 if (SimplifyDemandedBits(
Op.getOperand(1),
DemandedBits, DemandedElts,
1660 Known2, TLO,
Depth + 1))
1673 if (SimplifyDemandedBits(
Op.getOperand(2),
DemandedBits, DemandedElts,
1674 Known, TLO,
Depth + 1))
1676 if (SimplifyDemandedBits(
Op.getOperand(1),
DemandedBits, DemandedElts,
1677 Known2, TLO,
Depth + 1))
1686 if (SimplifyDemandedBits(
Op.getOperand(3),
DemandedBits, DemandedElts,
1687 Known, TLO,
Depth + 1))
1689 if (SimplifyDemandedBits(
Op.getOperand(2),
DemandedBits, DemandedElts,
1690 Known2, TLO,
Depth + 1))
1735 if (
const APInt *SA =
1737 unsigned ShAmt = SA->getZExtValue();
1747 if (
const APInt *SA2 =
1749 unsigned C1 = SA2->getZExtValue();
1751 int Diff = ShAmt - C1;
1770 if (ShAmt < InnerBits &&
DemandedBits.getActiveBits() <= InnerBits &&
1771 isTypeDesirableForOp(
ISD::SHL, InnerVT)) {
1787 if (
const APInt *SA2 =
1789 unsigned InnerShAmt = SA2->getZExtValue();
1790 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1792 (InnerBits - InnerShAmt + ShAmt) &&
1806 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1809 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1812 Flags.setNoSignedWrap(
false);
1813 Flags.setNoUnsignedWrap(
false);
1814 Op->setFlags(Flags);
1819 Known.
Zero <<= ShAmt;
1820 Known.
One <<= ShAmt;
1826 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1827 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1848 isTypeDesirableForOp(
ISD::SHL, HalfVT) &&
1857 Flags.setNoSignedWrap(IsNSW);
1858 Flags.setNoUnsignedWrap(IsNUW);
1863 NewShiftAmt, Flags);
1876 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
1879 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1882 Flags.setNoSignedWrap(
false);
1883 Flags.setNoUnsignedWrap(
false);
1884 Op->setFlags(Flags);
1894 if (
const APInt *MaxSA =
1896 unsigned ShAmt = MaxSA->getZExtValue();
1897 unsigned NumSignBits =
1900 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1912 DemandedElts,
Depth + 1))
1915 if (
const APInt *SA =
1917 unsigned ShAmt = SA->getZExtValue();
1927 if (
const APInt *SA2 =
1929 unsigned C1 = SA2->getZExtValue();
1931 int Diff = ShAmt - C1;
1947 if (
Op->getFlags().hasExact())
1956 isTypeDesirableForOp(
ISD::SRL, HalfVT) &&
1972 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1983 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1984 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
2018 DemandedElts,
Depth + 1))
2021 if (
const APInt *SA =
2023 unsigned ShAmt = SA->getZExtValue();
2030 if (
const APInt *InnerSA =
2032 unsigned LowBits =
BitWidth - ShAmt;
2038 if (*InnerSA == ShAmt) {
2048 unsigned NumSignBits =
2050 if (NumSignBits > ShAmt)
2060 if (
Op->getFlags().hasExact())
2068 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
2080 Flags.setExact(
Op->getFlags().hasExact());
2098 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2099 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
2116 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2121 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1,
DemandedBits, DemandedElts,
2122 Known, TLO,
Depth + 1))
2131 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2134 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2147 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2148 Op0, Demanded0, DemandedElts, TLO.
DAG,
Depth + 1);
2149 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2150 Op1, Demanded1, DemandedElts, TLO.
DAG,
Depth + 1);
2151 if (DemandedOp0 || DemandedOp1) {
2152 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2153 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2164 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
2165 Known2, TLO,
Depth + 1))
2181 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2187 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2197 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2202 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2211 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2221 unsigned Opc =
Op.getOpcode();
2228 unsigned NumSignBits =
2232 if (NumSignBits >= NumDemandedUpperBits)
2273 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2299 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2307 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2327 EVT ExVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2332 unsigned MinSignedBits =
2334 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2337 if (!AlreadySignExtended) {
2355 InputDemandedBits.
setBit(ExVTBits - 1);
2357 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2366 if (Known.
Zero[ExVTBits - 1])
2370 if (Known.
One[ExVTBits - 1]) {
2380 EVT HalfVT =
Op.getOperand(0).getValueType();
2388 if (SimplifyDemandedBits(
Op.getOperand(0), MaskLo, KnownLo, TLO,
Depth + 1))
2391 if (SimplifyDemandedBits(
Op.getOperand(1), MaskHi, KnownHi, TLO,
Depth + 1))
2394 Known = KnownHi.
concat(KnownLo);
2403 EVT SrcVT = Src.getValueType();
2412 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2424 APInt InDemandedElts = DemandedElts.
zext(InElts);
2425 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2427 if (Flags.hasNonNeg()) {
2428 Flags.setNonNeg(
false);
2429 Op->setFlags(Flags);
2438 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2439 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2449 EVT SrcVT = Src.getValueType();
2454 APInt InDemandedElts = DemandedElts.
zext(InElts);
2459 InDemandedBits.
setBit(InBits - 1);
2465 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2480 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2496 Flags.setNonNeg(
true);
2502 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2503 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2513 EVT SrcVT = Src.getValueType();
2520 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2525 APInt InDemandedElts = DemandedElts.
zext(InElts);
2526 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2534 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2535 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2544 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2546 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2552 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2553 Src, TruncMask, DemandedElts, TLO.
DAG,
Depth + 1))
2558 switch (Src.getOpcode()) {
2569 if (Src.getNode()->hasOneUse()) {
2570 const APInt *ShAmtC =
2601 EVT ZVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2603 if (SimplifyDemandedBits(
Op.getOperand(0), ~InMask |
DemandedBits, Known,
2608 Known.
Zero |= ~InMask;
2609 Known.
One &= (~Known.Zero);
2615 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2616 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2624 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx))
2625 if (CIdx->getAPIntValue().ult(NumSrcElts))
2632 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
2634 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2640 if (
SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2641 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
2657 EVT SrcVT = Src.getValueType();
2667 if ((OpVTLegal || i32Legal) && VT.
isSimple() && SrcVT != MVT::f16 &&
2668 SrcVT != MVT::f128) {
2670 EVT Ty = OpVTLegal ? VT : MVT::i32;
2674 unsigned OpVTSizeInBits =
Op.getValueSizeInBits();
2675 if (!OpVTLegal && OpVTSizeInBits > 32)
2677 unsigned ShVal =
Op.getValueSizeInBits() - 1;
2687 unsigned Scale =
BitWidth / NumSrcEltBits;
2691 for (
unsigned i = 0; i != Scale; ++i) {
2692 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2693 unsigned BitOffset = EltOffset * NumSrcEltBits;
2696 DemandedSrcBits |= Sub;
2697 for (
unsigned j = 0; j != NumElts; ++j)
2698 if (DemandedElts[j])
2699 DemandedSrcElts.
setBit((j * Scale) + i);
2703 APInt KnownSrcUndef, KnownSrcZero;
2704 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2705 KnownSrcZero, TLO,
Depth + 1))
2709 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2710 KnownSrcBits, TLO,
Depth + 1))
2712 }
else if (IsLE && (NumSrcEltBits %
BitWidth) == 0) {
2714 unsigned Scale = NumSrcEltBits /
BitWidth;
2718 for (
unsigned i = 0; i != NumElts; ++i)
2719 if (DemandedElts[i]) {
2722 DemandedSrcElts.
setBit(i / Scale);
2726 APInt KnownSrcUndef, KnownSrcZero;
2727 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2728 KnownSrcZero, TLO,
Depth + 1))
2733 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2734 KnownSrcBits, TLO,
Depth + 1))
2739 if (
SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2740 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
2762 if (
C &&
C->getAPIntValue().countr_zero() == CTZ) {
2781 SDValue Op0 =
Op.getOperand(0), Op1 =
Op.getOperand(1);
2786 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, KnownOp0, TLO,
2788 SimplifyDemandedBits(Op1, LoMask, DemandedElts, KnownOp1, TLO,
2792 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2795 Flags.setNoSignedWrap(
false);
2796 Flags.setNoUnsignedWrap(
false);
2797 Op->setFlags(Flags);
2809 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2810 Op0, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2811 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2812 Op1, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2813 if (DemandedOp0 || DemandedOp1) {
2814 Flags.setNoSignedWrap(
false);
2815 Flags.setNoUnsignedWrap(
false);
2816 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2817 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2819 TLO.
DAG.
getNode(
Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2831 if (
C && !
C->isAllOnes() && !
C->isOne() &&
2832 (
C->getAPIntValue() | HighMask).isAllOnes()) {
2836 Flags.setNoSignedWrap(
false);
2837 Flags.setNoUnsignedWrap(
false);
2845 auto getShiftLeftAmt = [&HighMask](
SDValue Mul) ->
unsigned {
2872 if (
unsigned ShAmt = getShiftLeftAmt(Op0))
2875 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2876 return foldMul(
ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
2880 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2881 return foldMul(
ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
2889 Op.getOpcode() ==
ISD::ADD, Flags.hasNoSignedWrap(),
2890 Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1);
2900 if (
Op.getValueType().isScalableVector())
2902 if (SimplifyDemandedBitsForTargetNode(
Op,
DemandedBits, DemandedElts,
2915 if (!isTargetCanonicalConstantNode(
Op) &&
2944 const APInt &DemandedElts,
2950 APInt KnownUndef, KnownZero;
2952 SimplifyDemandedVectorElts(
Op, DemandedElts, KnownUndef, KnownZero, TLO);
2964 const APInt &UndefOp0,
2965 const APInt &UndefOp1) {
2968 "Vector binop only");
2973 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
2975 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
2976 const APInt &UndefVals) {
2977 if (UndefVals[
Index])
2980 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2984 auto *
C = dyn_cast<ConstantSDNode>(Elt);
2985 if (isa<ConstantFPSDNode>(Elt) || Elt.
isUndef() || (
C && !
C->isOpaque()))
2993 for (
unsigned i = 0; i != NumElts; ++i) {
3012 bool AssumeSingleUse)
const {
3013 EVT VT =
Op.getValueType();
3014 unsigned Opcode =
Op.getOpcode();
3015 APInt DemandedElts = OriginalDemandedElts;
3030 "Mask size mismatches value type element count!");
3039 if (!AssumeSingleUse && !
Op.getNode()->hasOneUse())
3043 if (DemandedElts == 0) {
3058 auto SimplifyDemandedVectorEltsBinOp = [&](
SDValue Op0,
SDValue Op1) {
3059 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
3061 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
3063 if (NewOp0 || NewOp1) {
3066 NewOp1 ? NewOp1 : Op1,
Op->getFlags());
3074 if (!DemandedElts[0]) {
3082 EVT SrcVT = Src.getValueType();
3094 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3104 EVT SrcVT = Src.getValueType();
3113 if (NumSrcElts == NumElts)
3114 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
3115 KnownZero, TLO,
Depth + 1);
3117 APInt SrcDemandedElts, SrcZero, SrcUndef;
3121 if ((NumElts % NumSrcElts) == 0) {
3122 unsigned Scale = NumElts / NumSrcElts;
3124 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3134 for (
unsigned i = 0; i != NumElts; ++i)
3135 if (DemandedElts[i]) {
3136 unsigned Ofs = (i % Scale) * EltSizeInBits;
3137 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
3141 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
3149 for (
unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3153 for (
unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3154 unsigned Elt = Scale * SrcElt + SubElt;
3155 if (DemandedElts[Elt])
3163 for (
unsigned i = 0; i != NumSrcElts; ++i) {
3164 if (SrcDemandedElts[i]) {
3166 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
3168 KnownUndef.
setBits(i * Scale, (i + 1) * Scale);
3176 if ((NumSrcElts % NumElts) == 0) {
3177 unsigned Scale = NumSrcElts / NumElts;
3179 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3185 for (
unsigned i = 0; i != NumElts; ++i) {
3186 if (DemandedElts[i]) {
3215 [&](
SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3217 bool Updated =
false;
3218 for (
unsigned i = 0; i != NumElts; ++i) {
3219 if (!DemandedElts[i] && !Ops[i].
isUndef()) {
3229 for (
unsigned i = 0; i != NumElts; ++i) {
3231 if (
SrcOp.isUndef()) {
3233 }
else if (EltSizeInBits ==
SrcOp.getScalarValueSizeInBits() &&
3241 EVT SubVT =
Op.getOperand(0).getValueType();
3244 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3247 APInt SubUndef, SubZero;
3248 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3251 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
3252 KnownZero.
insertBits(SubZero, i * NumSubElts);
3257 bool FoundNewSub =
false;
3259 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3262 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3263 SubOp, SubElts, TLO.
DAG,
Depth + 1);
3264 DemandedSubOps.
push_back(NewSubOp ? NewSubOp : SubOp);
3265 FoundNewSub = NewSubOp ?
true : FoundNewSub;
3283 APInt DemandedSrcElts = DemandedElts;
3286 APInt SubUndef, SubZero;
3287 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3292 if (!DemandedSrcElts && !Src.isUndef())
3297 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3305 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3306 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3307 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3308 Sub, DemandedSubElts, TLO.
DAG,
Depth + 1);
3309 if (NewSrc || NewSub) {
3310 NewSrc = NewSrc ? NewSrc : Src;
3311 NewSub = NewSub ? NewSub : Sub;
3313 NewSub,
Op.getOperand(2));
3322 if (Src.getValueType().isScalableVector())
3325 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3328 APInt SrcUndef, SrcZero;
3329 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3337 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3338 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3350 auto *CIdx = dyn_cast<ConstantSDNode>(
Op.getOperand(2));
3354 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3355 unsigned Idx = CIdx->getZExtValue();
3356 if (!DemandedElts[
Idx])
3359 APInt DemandedVecElts(DemandedElts);
3361 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3362 KnownZero, TLO,
Depth + 1))
3371 APInt VecUndef, VecZero;
3372 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3385 APInt UndefSel, ZeroSel;
3386 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO,
3391 APInt DemandedLHS(DemandedElts);
3392 APInt DemandedRHS(DemandedElts);
3393 APInt UndefLHS, ZeroLHS;
3394 APInt UndefRHS, ZeroRHS;
3395 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3398 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3402 KnownUndef = UndefLHS & UndefRHS;
3403 KnownZero = ZeroLHS & ZeroRHS;
3407 APInt DemandedSel = DemandedElts & ~KnownZero;
3408 if (DemandedSel != DemandedElts)
3409 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO,
3421 APInt DemandedLHS(NumElts, 0);
3422 APInt DemandedRHS(NumElts, 0);
3423 for (
unsigned i = 0; i != NumElts; ++i) {
3424 int M = ShuffleMask[i];
3425 if (M < 0 || !DemandedElts[i])
3427 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
3428 if (M < (
int)NumElts)
3431 DemandedRHS.
setBit(M - NumElts);
3435 APInt UndefLHS, ZeroLHS;
3436 APInt UndefRHS, ZeroRHS;
3437 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3440 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3445 bool Updated =
false;
3446 bool IdentityLHS =
true, IdentityRHS =
true;
3448 for (
unsigned i = 0; i != NumElts; ++i) {
3449 int &M = NewMask[i];
3452 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
3453 (M >= (
int)NumElts && UndefRHS[M - NumElts])) {
3457 IdentityLHS &= (M < 0) || (M == (
int)i);
3458 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3463 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.
LegalOps) {
3465 buildLegalVectorShuffle(VT,
DL,
LHS,
RHS, NewMask, TLO.
DAG);
3471 for (
unsigned i = 0; i != NumElts; ++i) {
3472 int M = ShuffleMask[i];
3475 }
else if (M < (
int)NumElts) {
3481 if (UndefRHS[M - NumElts])
3483 if (ZeroRHS[M - NumElts])
3492 APInt SrcUndef, SrcZero;
3494 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3495 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3496 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3503 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3504 DemandedSrcElts == 1) {
3517 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() ==
ISD::AND &&
3518 Op->isOnlyUserOf(Src.getNode()) &&
3519 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3521 EVT SrcVT = Src.getValueType();
3528 ISD::AND,
DL, SrcVT, {Src.getOperand(1), Mask})) {
3542 if (Op0 == Op1 &&
Op->isOnlyUserOf(Op0.
getNode())) {
3543 APInt UndefLHS, ZeroLHS;
3544 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3565 APInt UndefRHS, ZeroRHS;
3566 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3569 APInt UndefLHS, ZeroLHS;
3570 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3574 KnownZero = ZeroLHS & ZeroRHS;
3580 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3592 APInt UndefRHS, ZeroRHS;
3593 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3596 APInt UndefLHS, ZeroLHS;
3597 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3601 KnownZero = ZeroLHS;
3602 KnownUndef = UndefLHS & UndefRHS;
3607 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3618 APInt SrcUndef, SrcZero;
3619 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
3624 APInt DemandedElts0 = DemandedElts & ~SrcZero;
3625 if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero,
3629 KnownUndef &= DemandedElts0;
3630 KnownZero &= DemandedElts0;
3635 if (DemandedElts.
isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
3642 KnownZero |= SrcZero;
3643 KnownUndef &= SrcUndef;
3644 KnownUndef &= ~KnownZero;
3648 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3655 if (SimplifyDemandedVectorElts(
Op.getOperand(0), DemandedElts, KnownUndef,
3656 KnownZero, TLO,
Depth + 1))
3668 if (SimplifyDemandedVectorEltsForTargetNode(
Op, DemandedElts, KnownUndef,
3669 KnownZero, TLO,
Depth))
3674 if (SimplifyDemandedBits(
Op,
DemandedBits, OriginalDemandedElts, Known,
3675 TLO,
Depth, AssumeSingleUse))
3681 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
3695 const APInt &DemandedElts,
3697 unsigned Depth)
const {
3702 "Should use MaskedValueIsZero if you don't know whether Op"
3703 " is a target node!");
3710 unsigned Depth)
const {
3722 unsigned Depth)
const {
3731 unsigned Depth)
const {
3736 "Should use ComputeNumSignBits if you don't know whether Op"
3737 " is a target node!");
3754 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3755 " is a target node!");
3766 "Should use SimplifyDemandedBits if you don't know whether Op"
3767 " is a target node!");
3768 computeKnownBitsForTargetNode(
Op, Known, DemandedElts, TLO.
DAG,
Depth);
3780 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3781 " is a target node!");
3814 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3815 " is a target node!");
3819 return !canCreateUndefOrPoisonForTargetNode(
Op, DemandedElts, DAG,
PoisonOnly,
3822 return DAG.isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly,
3834 "Should use canCreateUndefOrPoison if you don't know whether Op"
3835 " is a target node!");
3843 unsigned Depth)
const {
3848 "Should use isKnownNeverNaN if you don't know whether Op"
3849 " is a target node!");
3854 const APInt &DemandedElts,
3857 unsigned Depth)
const {
3862 "Should use isSplatValue if you don't know whether Op"
3863 " is a target node!");
3878 CVal = CN->getAPIntValue();
3879 EltWidth =
N.getValueType().getScalarSizeInBits();
3886 CVal = CVal.
trunc(EltWidth);
3892 return CVal.
isOne();
3934 return (
N->isOne() && !SExt) || (SExt && (
N->getValueType(0) != MVT::i1));
3937 return N->isAllOnes() && SExt;
3946 DAGCombinerInfo &DCI)
const {
3974 auto *AndC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
3975 if (AndC &&
isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
3978 AndC->getAPIntValue().getActiveBits());
4005 if (isXAndYEqZeroPreferableToXAndYEqY(
Cond, OpVT) &&
4013 if (DCI.isBeforeLegalizeOps() ||
4047SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4052 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
4061 if (!(C01 = dyn_cast<ConstantSDNode>(N0->
getOperand(1))))
4065 EVT XVT =
X.getValueType();
4089 auto checkConstants = [&
I1, &I01]() ->
bool {
4094 if (checkConstants()) {
4102 if (!checkConstants())
4108 const unsigned KeptBits =
I1.logBase2();
4109 const unsigned KeptBitsMinusOne = I01.
logBase2();
4112 if (KeptBits != (KeptBitsMinusOne + 1))
4127 return DAG.
getSetCC(
DL, SCCVT, SExtInReg,
X, NewCond);
4131SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4133 DAGCombinerInfo &DCI,
const SDLoc &
DL)
const {
4135 "Should be a comparison with 0.");
4137 "Valid only for [in]equality comparisons.");
4139 unsigned NewShiftOpcode;
4150 unsigned OldShiftOpcode =
V.getOpcode();
4151 switch (OldShiftOpcode) {
4163 C =
V.getOperand(0);
4168 Y =
V.getOperand(1);
4172 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4173 X, XC,
CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG);
4190 EVT VT =
X.getValueType();
4205 DAGCombinerInfo &DCI)
const {
4208 "Unexpected binop");
4237 if (!DCI.isCalledByLegalizer())
4238 DCI.AddToWorklist(YShl1.
getNode());
4253 if (CTPOP.getOpcode() !=
ISD::CTPOP || !CTPOP.hasOneUse())
4256 EVT CTVT = CTPOP.getValueType();
4257 SDValue CTOp = CTPOP.getOperand(0);
4277 for (
unsigned i = 0; i <
Passes; i++) {
4326 auto getRotateSource = [](
SDValue X) {
4328 return X.getOperand(0);
4335 if (
SDValue R = getRotateSource(N0))
4368 if (!C1 || !C1->
isZero())
4377 if (!ShAmtC || ShAmtC->getAPIntValue().uge(
BitWidth))
4381 unsigned ShAmt = ShAmtC->getZExtValue();
4390 if (
Or.getOperand(0) ==
Other) {
4391 X =
Or.getOperand(0);
4392 Y =
Or.getOperand(1);
4395 if (
Or.getOperand(1) ==
Other) {
4396 X =
Or.getOperand(1);
4397 Y =
Or.getOperand(0);
4407 if (matchOr(F0, F1)) {
4414 if (matchOr(F1, F0)) {
4430 const SDLoc &dl)
const {
4440 bool N0ConstOrSplat =
4442 bool N1ConstOrSplat =
4450 if (N0ConstOrSplat && !N1ConstOrSplat &&
4453 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4459 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4464 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4473 const APInt &C1 = N1C->getAPIntValue();
4493 return DAG.
getNode(LogicOp, dl, VT, IsXZero, IsYZero);
4523 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
4524 const APInt &C1 = N1C->getAPIntValue();
4539 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->
getOperand(1)))
4540 if ((
C->getAPIntValue()+1).isPowerOf2()) {
4541 MinBits =
C->getAPIntValue().countr_one();
4549 }
else if (
auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
4552 MinBits = LN0->getMemoryVT().getSizeInBits();
4556 MinBits = LN0->getMemoryVT().getSizeInBits();
4567 MinBits >= ReqdBits) {
4569 if (isTypeDesirableForOp(
ISD::SETCC, MinVT)) {
4572 if (MinBits == 1 && C1 == 1)
4591 if (TopSetCC.
getValueType() == MVT::i1 && VT == MVT::i1 &&
4604 cast<CondCodeSDNode>(TopSetCC.
getOperand(2))->get(),
4625 unsigned bestWidth = 0, bestOffset = 0;
4631 unsigned maskWidth = origWidth;
4639 for (
unsigned width = 8; width < origWidth; width *= 2) {
4646 unsigned maxOffset = origWidth - width;
4647 for (
unsigned offset = 0; offset <= maxOffset; offset += 8) {
4648 if (Mask.isSubsetOf(newMask)) {
4649 unsigned ptrOffset =
4651 unsigned IsFast = 0;
4657 bestOffset = ptrOffset / 8;
4658 bestMask = Mask.lshr(offset);
4672 if (bestOffset != 0)
4754 ExtDstTy != ExtSrcTy &&
"Unexpected types!");
4761 return DAG.
getSetCC(dl, VT, ZextOp,
4763 }
else if ((N1C->isZero() || N1C->isOne()) &&
4810 return DAG.
getSetCC(dl, VT, Val, N1,
4813 }
else if (N1C->isOne()) {
4850 cast<VTSDNode>(Op0.
getOperand(1))->getVT() == MVT::i1)
4874 N1C && N1C->isAllOnes()) {
4881 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1,
Cond, DCI, dl))
4888 const APInt &C1 = N1C->getAPIntValue();
4890 APInt MinVal, MaxVal;
4912 (!N1C->isOpaque() || (
C.getBitWidth() <= 64 &&
4932 (!N1C->isOpaque() || (
C.getBitWidth() <= 64 &&
4980 if (
SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4981 VT, N0, N1,
Cond, DCI, dl))
4988 bool CmpZero = N1C->isZero();
4989 bool CmpNegOne = N1C->isAllOnes();
4990 if ((CmpZero || CmpNegOne) && N0.
hasOneUse()) {
4993 unsigned EltBits = V.getScalarValueSizeInBits();
4994 if (V.getOpcode() !=
ISD::OR || (EltBits % 2) != 0)
5001 isa<ConstantSDNode>(
RHS.getOperand(1)) &&
5002 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5005 Hi =
RHS.getOperand(0);
5009 isa<ConstantSDNode>(
LHS.getOperand(1)) &&
5010 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5013 Hi =
LHS.getOperand(0);
5021 unsigned HalfBits = EltBits / 2;
5032 if (IsConcat(N0,
Lo,
Hi))
5033 return MergeConcat(
Lo,
Hi);
5070 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
5072 const APInt &C1 = N1C->getAPIntValue();
5084 if (
auto *AndRHS = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
5087 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5088 if (AndRHS->getAPIntValue().isPowerOf2() &&
5096 }
else if (
Cond ==
ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5117 if (
auto *AndRHS = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
5118 const APInt &AndRHSC = AndRHS->getAPIntValue();
5157 return DAG.
getSetCC(dl, VT, Shift, CmpRHS, NewCond);
5163 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
5164 auto *CFP = cast<ConstantFPSDNode>(N1);
5165 assert(!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value");
5186 !
isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(0))) {
5205 if (CFP->getValueAPF().isInfinity()) {
5206 bool IsNegInf = CFP->getValueAPF().isNegative();
5217 return DAG.
getSetCC(dl, VT, N0, N1, NewCond);
5226 "Integer types should be handled by FoldSetCC");
5232 if (UOF ==
unsigned(EqTrue))
5237 if (NewCond !=
Cond &&
5240 return DAG.
getSetCC(dl, VT, N0, N1, NewCond);
5247 if ((isSignedIntSetCC(
Cond) || isUnsignedIntSetCC(
Cond)) &&
5284 bool LegalRHSImm =
false;
5286 if (
auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
5287 if (
auto *LHSR = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
5292 DAG.
getConstant(RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5300 DAG.
getConstant(LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5306 if (
auto *SUBC = dyn_cast<ConstantSDNode>(N0.
getOperand(0)))
5310 DAG.
getConstant(SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5315 if (RHSC->getValueType(0).getSizeInBits() <= 64)
5324 if (
SDValue V = foldSetCCWithBinOp(VT, N0, N1,
Cond, dl, DCI))
5330 if (
SDValue V = foldSetCCWithBinOp(VT, N1, N0,
Cond, dl, DCI))
5333 if (
SDValue V = foldSetCCWithAnd(VT, N0, N1,
Cond, dl, DCI))
5344 if (
SDValue Folded = buildUREMEqFold(VT, N0, N1,
Cond, DCI, dl))
5347 if (
SDValue Folded = buildSREMEqFold(VT, N0, N1,
Cond, DCI, dl))
5360 N0 = DAG.
getNOT(dl, Temp, OpVT);
5369 Temp = DAG.
getNOT(dl, N0, OpVT);
5376 Temp = DAG.
getNOT(dl, N1, OpVT);
5383 Temp = DAG.
getNOT(dl, N0, OpVT);
5390 Temp = DAG.
getNOT(dl, N1, OpVT);
5399 N0 = DAG.
getNode(ExtendCode, dl, VT, N0);
5415 if (
auto *GASD = dyn_cast<GlobalAddressSDNode>(
N)) {
5416 GA = GASD->getGlobal();
5417 Offset += GASD->getOffset();
5425 if (
auto *V = dyn_cast<ConstantSDNode>(N2)) {
5426 Offset += V->getSExtValue();
5430 if (
auto *V = dyn_cast<ConstantSDNode>(N1)) {
5431 Offset += V->getSExtValue();
5452 unsigned S = Constraint.
size();
5455 switch (Constraint[0]) {
5458 return C_RegisterClass;
5486 if (S > 1 && Constraint[0] ==
'{' && Constraint[S - 1] ==
'}') {
5487 if (S == 8 && Constraint.
substr(1, 6) ==
"memory")
5515 std::vector<SDValue> &Ops,
5518 if (Constraint.
size() > 1)
5521 char ConstraintLetter = Constraint[0];
5522 switch (ConstraintLetter) {
5538 if ((
C = dyn_cast<ConstantSDNode>(
Op)) && ConstraintLetter !=
's') {
5542 bool IsBool =
C->getConstantIntValue()->getBitWidth() == 1;
5552 if (ConstraintLetter !=
'n') {
5553 if (
const auto *GA = dyn_cast<GlobalAddressSDNode>(
Op)) {
5555 GA->getValueType(0),
5556 Offset + GA->getOffset()));
5559 if (
const auto *BA = dyn_cast<BlockAddressSDNode>(
Op)) {
5561 BA->getBlockAddress(), BA->getValueType(0),
5562 Offset + BA->getOffset(), BA->getTargetFlags()));
5565 if (isa<BasicBlockSDNode>(
Op)) {
5570 const unsigned OpCode =
Op.getOpcode();
5572 if ((
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0))))
5573 Op =
Op.getOperand(1);
5576 (
C = dyn_cast<ConstantSDNode>(
Op.getOperand(1))))
5577 Op =
Op.getOperand(0);
5594std::pair<unsigned, const TargetRegisterClass *>
5600 assert(*(Constraint.
end() - 1) ==
'}' &&
"Not a brace enclosed constraint?");
5605 std::pair<unsigned, const TargetRegisterClass *> R =
5617 std::pair<unsigned, const TargetRegisterClass *> S =
5618 std::make_pair(PR, RC);
5640 assert(!ConstraintCode.empty() &&
"No known constraint!");
5641 return isdigit(
static_cast<unsigned char>(ConstraintCode[0]));
5647 assert(!ConstraintCode.empty() &&
"No known constraint!");
5648 return atoi(ConstraintCode.c_str());
5662 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
5663 unsigned maCount = 0;
5669 unsigned LabelNo = 0;
5672 ConstraintOperands.emplace_back(std::move(CI));
5682 switch (OpInfo.
Type) {
5692 assert(!Call.getType()->isVoidTy() &&
"Bad inline asm!");
5693 if (
StructType *STy = dyn_cast<StructType>(Call.getType())) {
5697 assert(ResNo == 0 &&
"Asm only has one result!");
5707 OpInfo.
CallOperandVal = cast<CallBrInst>(&Call)->getIndirectDest(LabelNo);
5718 OpTy = Call.getParamElementType(ArgNo);
5719 assert(OpTy &&
"Indirect operand must have elementtype attribute");
5723 if (
StructType *STy = dyn_cast<StructType>(OpTy))
5724 if (STy->getNumElements() == 1)
5725 OpTy = STy->getElementType(0);
5730 unsigned BitSize =
DL.getTypeSizeInBits(OpTy);
5751 if (!ConstraintOperands.empty()) {
5753 unsigned bestMAIndex = 0;
5754 int bestWeight = -1;
5760 for (maIndex = 0; maIndex < maCount; ++maIndex) {
5762 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
5763 cIndex != eIndex; ++cIndex) {
5784 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
5789 weightSum += weight;
5792 if (weightSum > bestWeight) {
5793 bestWeight = weightSum;
5794 bestMAIndex = maIndex;
5801 cInfo.selectAlternative(bestMAIndex);
5806 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
5807 cIndex != eIndex; ++cIndex) {
5818 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
5821 std::pair<unsigned, const TargetRegisterClass *> InputRC =
5826 (MatchRC.second != InputRC.second)) {
5828 " with a matching output constraint of"
5829 " incompatible type!");
5835 return ConstraintOperands;
5870 if (maIndex >= (
int)
info.multipleAlternatives.size())
5871 rCodes = &
info.Codes;
5873 rCodes = &
info.multipleAlternatives[maIndex].Codes;
5877 for (
const std::string &rCode : *rCodes) {
5879 getSingleConstraintMatchWeight(
info, rCode.c_str());
5880 if (weight > BestWeight)
5881 BestWeight = weight;
5894 Value *CallOperandVal =
info.CallOperandVal;
5897 if (!CallOperandVal)
5900 switch (*constraint) {
5903 if (isa<ConstantInt>(CallOperandVal))
5904 weight = CW_Constant;
5907 if (isa<GlobalValue>(CallOperandVal))
5908 weight = CW_Constant;
5912 if (isa<ConstantFP>(CallOperandVal))
5913 weight = CW_Constant;
5926 weight = CW_Register;
5930 weight = CW_Default;
5964 Ret.reserve(OpInfo.
Codes.size());
5979 Ret.emplace_back(Code, CType);
5984 return getConstraintPiority(a.second) > getConstraintPiority(b.second);
5998 "need immediate or other");
6003 std::vector<SDValue> ResultOps;
6005 return !ResultOps.empty();
6013 assert(!OpInfo.
Codes.empty() &&
"Must have at least one constraint");
6016 if (OpInfo.
Codes.size() == 1) {
6024 unsigned BestIdx = 0;
6025 for (
const unsigned E =
G.size();
6032 if (BestIdx + 1 == E) {
6048 if (isa<ConstantInt>(v) || isa<Function>(v)) {
6052 if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
6059 if (
const char *Repl = LowerXConstraint(OpInfo.
ConstraintVT)) {
6073 EVT VT =
N->getValueType(0);
6078 bool UseSRA =
false;
6084 APInt Divisor =
C->getAPIntValue();
6106 "Expected matchUnaryPredicate to return one element for scalable "
6111 assert(isa<ConstantSDNode>(Op1) &&
"Expected a constant");
6113 Factor = Factors[0];
6122 Flags.setExact(
true);
6163 EVT VT =
N->getValueType(0);
6199 bool IsAfterLegalization,
6202 EVT VT =
N->getValueType(0);
6228 if (
N->getFlags().hasExact())
6237 const APInt &Divisor =
C->getAPIntValue();
6239 int NumeratorFactor = 0;
6250 NumeratorFactor = 1;
6253 NumeratorFactor = -1;
6270 SDValue MagicFactor, Factor, Shift, ShiftMask;
6278 Shifts.
size() == 1 && ShiftMasks.
size() == 1 &&
6279 "Expected matchUnaryPredicate to return one element for scalable "
6286 assert(isa<ConstantSDNode>(N1) &&
"Expected a constant");
6287 MagicFactor = MagicFactors[0];
6288 Factor = Factors[0];
6290 ShiftMask = ShiftMasks[0];
6331 SDValue Q = GetMULHS(N0, MagicFactor);
6361 bool IsAfterLegalization,
6364 EVT VT =
N->getValueType(0);
6395 unsigned LeadingZeros = 0;
6396 if (!VT.
isVector() && isa<ConstantSDNode>(N1)) {
6404 bool UseNPQ =
false, UsePreShift =
false, UsePostShift =
false;
6410 const APInt& Divisor =
C->getAPIntValue();
6412 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
6416 if (Divisor.
isOne()) {
6417 PreShift = PostShift = DAG.
getUNDEF(ShSVT);
6418 MagicFactor = NPQFactor = DAG.
getUNDEF(SVT);
6426 "We shouldn't generate an undefined shift!");
6428 "We shouldn't generate an undefined shift!");
6430 "Unexpected pre-shift");
6437 UseNPQ |= magics.
IsAdd;
6438 UsePreShift |= magics.
PreShift != 0;
6453 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
6461 NPQFactors.
size() == 1 && PostShifts.
size() == 1 &&
6462 "Expected matchUnaryPredicate to return one for scalable vectors");
6468 assert(isa<ConstantSDNode>(N1) &&
"Expected a constant");
6469 PreShift = PreShifts[0];
6470 MagicFactor = MagicFactors[0];
6471 PostShift = PostShifts[0];
6518 Q = GetMULHU(Q, MagicFactor);
6531 NPQ = GetMULHU(NPQ, NPQFactor);
6550 return DAG.
getSelect(dl, VT, IsOne, N0, Q);
6559 std::function<
bool(
SDValue)> Predicate,
6564 if (SplatValue != Values.
end()) {
6567 return Value == *SplatValue || Predicate(
Value);
6569 Replacement = *SplatValue;
6573 if (!AlternativeReplacement)
6576 Replacement = AlternativeReplacement;
6578 std::replace_if(Values.
begin(), Values.
end(), Predicate, Replacement);
6589 DAGCombinerInfo &DCI,
6592 if (
SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode,
Cond,
6595 DCI.AddToWorklist(
N);
6603TargetLowering::prepareUREMEqFold(
EVT SETCCVT,
SDValue REMNode,
6605 DAGCombinerInfo &DCI,
const SDLoc &
DL,
6613 "Only applicable for (in)equality comparisons.");
6626 bool ComparingWithAllZeros =
true;
6627 bool AllComparisonsWithNonZerosAreTautological =
true;
6628 bool HadTautologicalLanes =
false;
6629 bool AllLanesAreTautological =
true;
6630 bool HadEvenDivisor =
false;
6631 bool AllDivisorsArePowerOfTwo =
true;
6632 bool HadTautologicalInvertedLanes =
false;
6641 const APInt &
Cmp = CCmp->getAPIntValue();
6643 ComparingWithAllZeros &=
Cmp.isZero();
6649 bool TautologicalInvertedLane =
D.ule(Cmp);
6650 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
6655 bool TautologicalLane =
D.isOne() || TautologicalInvertedLane;
6656 HadTautologicalLanes |= TautologicalLane;
6657 AllLanesAreTautological &= TautologicalLane;
6663 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
6666 unsigned K =
D.countr_zero();
6667 assert((!
D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.");
6671 HadEvenDivisor |= (
K != 0);
6674 AllDivisorsArePowerOfTwo &= D0.
isOne();
6678 unsigned W =
D.getBitWidth();
6680 assert((D0 *
P).isOne() &&
"Multiplicative inverse basic check failed.");
6693 "We are expecting that K is always less than all-ones for ShSVT");
6696 if (TautologicalLane) {
6720 if (AllLanesAreTautological)
6725 if (AllDivisorsArePowerOfTwo)
6730 if (HadTautologicalLanes) {
6745 "Expected matchBinaryPredicate to return one element for "
6756 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
6760 "Expecting that the types on LHS and RHS of comparisons match.");
6770 if (HadEvenDivisor) {
6783 if (!HadTautologicalInvertedLanes)
6789 assert(VT.
isVector() &&
"Can/should only get here for vectors.");
6796 SDValue TautologicalInvertedChannels =
6806 DL, SETCCVT, SETCCVT);
6808 Replacement, NewCC);
6816 TautologicalInvertedChannels);
6829 DAGCombinerInfo &DCI,
6832 if (
SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode,
Cond,
6834 assert(Built.
size() <= 7 &&
"Max size prediction failed.");
6836 DCI.AddToWorklist(
N);
6844TargetLowering::prepareSREMEqFold(
EVT SETCCVT,
SDValue REMNode,
6846 DAGCombinerInfo &DCI,
const SDLoc &
DL,
6871 "Only applicable for (in)equality comparisons.");
6887 if (!CompTarget || !CompTarget->
isZero())
6890 bool HadIntMinDivisor =
false;
6891 bool HadOneDivisor =
false;
6892 bool AllDivisorsAreOnes =
true;
6893 bool HadEvenDivisor =
false;
6894 bool NeedToApplyOffset =
false;
6895 bool AllDivisorsArePowerOfTwo =
true;
6910 HadIntMinDivisor |=
D.isMinSignedValue();
6913 HadOneDivisor |=
D.isOne();
6914 AllDivisorsAreOnes &=
D.isOne();
6917 unsigned K =
D.countr_zero();
6918 assert((!
D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.");
6921 if (!
D.isMinSignedValue()) {
6924 HadEvenDivisor |= (
K != 0);
6929 AllDivisorsArePowerOfTwo &= D0.
isOne();
6933 unsigned W =
D.getBitWidth();
6935 assert((D0 *
P).isOne() &&
"Multiplicative inverse basic check failed.");
6941 if (!
D.isMinSignedValue()) {
6944 NeedToApplyOffset |=
A != 0;
6951 "We are expecting that A is always less than all-ones for SVT");
6953 "We are expecting that K is always less than all-ones for ShSVT");
6991 if (AllDivisorsAreOnes)
6996 if (AllDivisorsArePowerOfTwo)
6999 SDValue PVal, AVal, KVal, QVal;
7001 if (HadOneDivisor) {
7021 QAmts.
size() == 1 &&
7022 "Expected matchUnaryPredicate to return one element for scalable "
7029 assert(isa<ConstantSDNode>(
D) &&
"Expected a constant");
7040 if (NeedToApplyOffset) {
7052 if (HadEvenDivisor) {
7067 if (!HadIntMinDivisor)
7073 assert(VT.
isVector() &&
"Can/should only get here for vectors.");
7108 MaskedIsZero, Fold);
7115 if (!isa<ConstantSDNode>(
Op.getOperand(0))) {
7117 "be a constant integer");
7127 EVT VT =
Op.getValueType();
7150 bool LegalOps,
bool OptForSize,
7152 unsigned Depth)
const {
7154 if (
Op.getOpcode() ==
ISD::FNEG ||
Op.getOpcode() == ISD::VP_FNEG) {
7156 return Op.getOperand(0);
7167 EVT VT =
Op.getValueType();
7168 unsigned Opcode =
Op.getOpcode();
7178 auto RemoveDeadNode = [&](
SDValue N) {
7179 if (
N &&
N.getNode()->use_empty())
7188 std::list<HandleSDNode> Handles;
7199 if (LegalOps && !IsOpLegal)
7202 APFloat V = cast<ConstantFPSDNode>(
Op)->getValueAPF();
7216 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
7224 return N.isUndef() ||
7229 if (LegalOps && !IsOpLegal)
7238 APFloat V = cast<ConstantFPSDNode>(
C)->getValueAPF();
7246 if (!
Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7257 getNegatedExpression(
X, DAG, LegalOps, OptForSize, CostX,
Depth);
7260 Handles.emplace_back(NegX);
7265 getNegatedExpression(
Y, DAG, LegalOps, OptForSize, CostY,
Depth);
7271 if (NegX && (CostX <= CostY)) {
7275 RemoveDeadNode(NegY);
7284 RemoveDeadNode(NegX);
7291 if (!
Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7313 getNegatedExpression(
X, DAG, LegalOps, OptForSize, CostX,
Depth);
7316 Handles.emplace_back(NegX);
7321 getNegatedExpression(
Y, DAG, LegalOps, OptForSize, CostY,
Depth);
7327 if (NegX && (CostX <= CostY)) {
7331 RemoveDeadNode(NegY);
7337 if (
C->isExactlyValue(2.0) &&
Op.getOpcode() ==
ISD::FMUL)
7345 RemoveDeadNode(NegX);
7352 if (!
Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7355 SDValue X =
Op.getOperand(0),
Y =
Op.getOperand(1), Z =
Op.getOperand(2);
7358 getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ,
Depth);
7364 Handles.emplace_back(NegZ);
7369 getNegatedExpression(
X, DAG, LegalOps, OptForSize, CostX,
Depth);
7372 Handles.emplace_back(NegX);
7377 getNegatedExpression(
Y, DAG, LegalOps, OptForSize, CostY,
Depth);
7383 if (NegX && (CostX <= CostY)) {
7384 Cost = std::min(CostX, CostZ);
7387 RemoveDeadNode(NegY);
7393 Cost = std::min(CostY, CostZ);
7396 RemoveDeadNode(NegX);
7404 if (
SDValue NegV = getNegatedExpression(
Op.getOperand(0), DAG, LegalOps,
7406 return DAG.
getNode(Opcode,
DL, VT, NegV);
7409 if (
SDValue NegV = getNegatedExpression(
Op.getOperand(0), DAG, LegalOps,
7420 getNegatedExpression(
LHS, DAG, LegalOps, OptForSize, CostLHS,
Depth);
7422 RemoveDeadNode(NegLHS);
7427 Handles.emplace_back(NegLHS);
7432 getNegatedExpression(
RHS, DAG, LegalOps, OptForSize, CostRHS,
Depth);
7440 RemoveDeadNode(NegLHS);
7441 RemoveDeadNode(NegRHS);
7445 Cost = std::min(CostLHS, CostRHS);
7446 return DAG.
getSelect(
DL, VT,
Op.getOperand(0), NegLHS, NegRHS);
7475 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
7488 if ((
Signed && HasSMUL_LOHI) || (!
Signed && HasUMUL_LOHI)) {
7516 if (MakeMUL_LOHI(LL, RL,
Lo,
Hi,
false)) {
7517 Result.push_back(
Lo);
7518 Result.push_back(
Hi);
7521 Result.push_back(Zero);
7522 Result.push_back(Zero);
7533 if (MakeMUL_LOHI(LL, RL,
Lo,
Hi,
true)) {
7534 Result.push_back(
Lo);
7535 Result.push_back(
Hi);
7540 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
7555 if (!MakeMUL_LOHI(LL, RL,
Lo,
Hi,
false))
7558 Result.push_back(
Lo);
7565 Result.push_back(
Hi);
7578 if (!MakeMUL_LOHI(LL, RH,
Lo,
Hi,
false))
7585 if (!MakeMUL_LOHI(LH, RL,
Lo,
Hi,
false))
7637 bool Ok = expandMUL_LOHI(
N->getOpcode(),
N->getValueType(0),
SDLoc(
N),
7638 N->getOperand(0),
N->getOperand(1), Result, HiLoVT,
7639 DAG, Kind, LL, LH, RL, RH);
7641 assert(Result.size() == 2);
7673 unsigned Opcode =
N->getOpcode();
7674 EVT VT =
N->getValueType(0);
7681 "Unexpected opcode");
7683 auto *CN = dyn_cast<ConstantSDNode>(
N->getOperand(1));
7687 APInt Divisor = CN->getAPIntValue();
7695 if (Divisor.
uge(HalfMaxPlus1))
7713 unsigned TrailingZeros = 0;
7727 if (HalfMaxPlus1.
urem(Divisor).
isOne()) {
7728 assert(!LL == !LH &&
"Expected both input halves or no input halves!");
7730 std::tie(LL, LH) = DAG.
SplitScalar(
N->getOperand(0), dl, HiLoVT, HiLoVT);
7734 if (TrailingZeros) {
7802 std::tie(QuotL, QuotH) = DAG.
SplitScalar(Quotient, dl, HiLoVT, HiLoVT);
7803 Result.push_back(QuotL);
7804 Result.push_back(QuotH);
7810 if (TrailingZeros) {
7816 Result.push_back(RemL);
7832 EVT VT =
Node->getValueType(0);
7842 bool IsFSHL =
Node->getOpcode() == ISD::VP_FSHL;
7845 EVT ShVT = Z.getValueType();
7851 ShAmt = DAG.
getNode(ISD::VP_UREM,
DL, ShVT, Z, BitWidthC, Mask, VL);
7852 InvShAmt = DAG.
getNode(ISD::VP_SUB,
DL, ShVT, BitWidthC, ShAmt, Mask, VL);
7853 ShX = DAG.
getNode(ISD::VP_SHL,
DL, VT,
X, IsFSHL ? ShAmt : InvShAmt, Mask,
7855 ShY = DAG.
getNode(ISD::VP_LSHR,
DL, VT,
Y, IsFSHL ? InvShAmt : ShAmt, Mask,
7863 ShAmt = DAG.
getNode(ISD::VP_AND,
DL, ShVT, Z, BitMask, Mask, VL);
7867 InvShAmt = DAG.
getNode(ISD::VP_AND,
DL, ShVT, NotZ, BitMask, Mask, VL);
7870 ShAmt = DAG.
getNode(ISD::VP_UREM,
DL, ShVT, Z, BitWidthC, Mask, VL);
7871 InvShAmt = DAG.
getNode(ISD::VP_SUB,
DL, ShVT, BitMask, ShAmt, Mask, VL);
7876 ShX = DAG.
getNode(ISD::VP_SHL,
DL, VT,
X, ShAmt, Mask, VL);
7878 ShY = DAG.
getNode(ISD::VP_LSHR,
DL, VT, ShY1, InvShAmt, Mask, VL);
7881 ShX = DAG.
getNode(ISD::VP_SHL,
DL, VT, ShX1, InvShAmt, Mask, VL);
7882 ShY = DAG.
getNode(ISD::VP_LSHR,
DL, VT,
Y, ShAmt, Mask, VL);
7885 return DAG.
getNode(ISD::VP_OR,
DL, VT, ShX, ShY, Mask, VL);
7890 if (Node->isVPOpcode())
7893 EVT VT = Node->getValueType(0);
7903 SDValue Z = Node->getOperand(2);
7906 bool IsFSHL = Node->getOpcode() ==
ISD::FSHL;
7909 EVT ShVT = Z.getValueType();
7979 EVT VT = Node->getValueType(0);
7981 bool IsLeft = Node->getOpcode() ==
ISD::ROTL;
7982 SDValue Op0 = Node->getOperand(0);
7983 SDValue Op1 = Node->getOperand(1);
7994 return DAG.
getNode(RevRot,
DL, VT, Op0, Sub);
7997 if (!AllowVectorOps && VT.
isVector() &&
8015 ShVal = DAG.
getNode(ShOpc,
DL, VT, Op0, ShAmt);
8017 HsVal = DAG.
getNode(HsOpc,
DL, VT, Op0, HsAmt);
8023 ShVal = DAG.
getNode(ShOpc,
DL, VT, Op0, ShAmt);
8034 assert(Node->getNumOperands() == 3 &&
"Not a double-shift!");
8035 EVT VT = Node->getValueType(0);
8041 SDValue ShOpLo = Node->getOperand(0);
8042 SDValue ShOpHi = Node->getOperand(1);
8043 SDValue ShAmt = Node->getOperand(2);
8086 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
8087 SDValue Src = Node->getOperand(OpNo);
8088 EVT SrcVT = Src.getValueType();
8089 EVT DstVT = Node->getValueType(0);
8093 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
8096 if (Node->isStrictFPOpcode())
8159 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
8160 SDValue Src = Node->getOperand(OpNo);
8162 EVT SrcVT = Src.getValueType();
8163 EVT DstVT = Node->getValueType(0);
8184 if (Node->isStrictFPOpcode()) {
8186 { Node->getOperand(0), Src });
8187 Chain = Result.getValue(1);
8201 if (Node->isStrictFPOpcode()) {
8203 Node->getOperand(0),
true);
8209 bool Strict = Node->isStrictFPOpcode() ||
8228 if (Node->isStrictFPOpcode()) {
8230 { Chain, Src, FltOfs });
8252 Result = DAG.
getSelect(dl, DstVT, Sel, True, False);
8263 if (Node->isStrictFPOpcode())
8266 SDValue Src = Node->getOperand(0);
8267 EVT SrcVT = Src.getValueType();
8268 EVT DstVT = Node->getValueType(0);
8291 llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT);
8311 unsigned Opcode = Node->getOpcode();
8316 if (Node->getFlags().hasNoNaNs()) {
8318 SDValue Op1 = Node->getOperand(0);
8319 SDValue Op2 = Node->getOperand(1);
8324 Flags.setNoSignedZeros(
true);
8337 EVT VT = Node->getValueType(0);
8341 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
8344 SDValue Quiet0 = Node->getOperand(0);
8345 SDValue Quiet1 = Node->getOperand(1);
8347 if (!Node->getFlags().hasNoNaNs()) {
8360 return DAG.
getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
8366 if ((Node->getFlags().hasNoNaNs() ||
8369 (Node->getFlags().hasNoSignedZeros() ||
8372 unsigned IEEE2018Op =
8375 return DAG.
getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
8376 Node->getOperand(1), Node->getFlags());
8379 if (
SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
8390 unsigned Opc =
N->getOpcode();
8391 EVT VT =
N->getValueType(0);
8407 bool MinMaxMustRespectOrderedZero =
false;
8411 MinMaxMustRespectOrderedZero =
true;
8422 if (!
N->getFlags().hasNoNaNs() &&
8431 if (!MinMaxMustRespectOrderedZero && !
N->getFlags().hasNoSignedZeros() &&
8457 bool IsOrdered = NanTest ==
fcNone;
8458 bool IsUnordered = NanTest ==
fcNan;
8461 if (!IsOrdered && !IsUnordered)
8462 return std::nullopt;
8464 if (OrderedMask ==
fcZero &&
8470 return std::nullopt;
8477 EVT OperandVT =
Op.getValueType();
8488 if (OperandVT == MVT::ppcf128) {
8491 OperandVT = MVT::f64;
8496 bool IsInverted =
false;
8499 Test = InvertedCheck;
8506 bool IsF80 = (ScalarFloatVT == MVT::f80);
8510 if (Flags.hasNoFPExcept() &&
8515 if (std::optional<bool> IsCmp0 =
8518 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
8525 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
8560 const unsigned ExplicitIntBitInF80 = 63;
8561 APInt ExpMask = Inf;
8563 ExpMask.
clearBit(ExplicitIntBitInF80);
8577 const auto appendResult = [&](
SDValue PartialRes) {
8587 const auto getIntBitIsSet = [&]() ->
SDValue {
8588 if (!IntBitIsSetV) {
8589 APInt IntBitMask(BitSize, 0);
8590 IntBitMask.
setBit(ExplicitIntBitInF80);
8595 return IntBitIsSetV;
8616 Test &= ~fcPosFinite;
8621 Test &= ~fcNegFinite;
8623 appendResult(PartialRes);
8632 appendResult(ExpIsZero);
8642 else if (PartialCheck ==
fcZero)
8646 appendResult(PartialRes);
8659 appendResult(PartialRes);
8662 if (
unsigned PartialCheck =
Test &
fcInf) {
8665 else if (PartialCheck ==
fcInf)
8672 appendResult(PartialRes);
8675 if (
unsigned PartialCheck =
Test &
fcNan) {
8676 APInt InfWithQnanBit = Inf | QNaNBitMask;
8678 if (PartialCheck ==
fcNan) {
8691 }
else if (PartialCheck ==
fcQNan) {
8703 appendResult(PartialRes);
8708 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
8711 APInt ExpLimit = ExpMask - ExpLSB;
8724 appendResult(PartialRes);
8747 EVT VT = Node->getValueType(0);
8754 if (!(Len <= 128 && Len % 8 == 0))
8813 for (
unsigned Shift = 8; Shift < Len; Shift *= 2) {
8824 EVT VT = Node->getValueType(0);
8827 SDValue Mask = Node->getOperand(1);
8828 SDValue VL = Node->getOperand(2);
8833 if (!(Len <= 128 && Len % 8 == 0))
8845 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5;
8848 Tmp1 = DAG.
getNode(ISD::VP_AND, dl, VT,
8852 Op = DAG.
getNode(ISD::VP_SUB, dl, VT,
Op, Tmp1, Mask, VL);
8855 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT,
Op, Mask33, Mask, VL);
8856 Tmp3 = DAG.
getNode(ISD::VP_AND, dl, VT,
8860 Op = DAG.
getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL);
8865 Tmp5 = DAG.
getNode(ISD::VP_ADD, dl, VT,
Op, Tmp4, Mask, VL);
8866 Op = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL);
8877 V = DAG.
getNode(ISD::VP_MUL, dl, VT,
Op, Mask01, Mask, VL);
8880 for (
unsigned Shift = 8; Shift < Len; Shift *= 2) {
8882 V = DAG.
getNode(ISD::VP_ADD, dl, VT, V,
8883 DAG.
getNode(ISD::VP_SHL, dl, VT, V, ShiftC, Mask, VL),
8887 return DAG.
getNode(ISD::VP_LSHR, dl, VT, V,
8893 EVT VT = Node->getValueType(0);
8932 for (
unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
8943 EVT VT = Node->getValueType(0);
8946 SDValue Mask = Node->getOperand(1);
8947 SDValue VL = Node->getOperand(2);
8957 for (
unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
8960 DAG.
getNode(ISD::VP_LSHR, dl, VT,
Op, Tmp, Mask, VL), Mask,
8965 return DAG.
getNode(ISD::VP_CTPOP, dl, VT,
Op, Mask, VL);
8974 :
APInt(64, 0x0218A392CD3D5DBFULL);
8988 for (
unsigned i = 0; i <
BitWidth; i++) {
9014 EVT VT = Node->getValueType(0);
9048 if (
SDValue V = CTTZTableLookup(Node, DAG, dl, VT,
Op, NumBitsPerElt))
9070 SDValue Mask = Node->getOperand(1);
9071 SDValue VL = Node->getOperand(2);
9073 EVT VT = Node->getValueType(0);
9080 SDValue Tmp = DAG.
getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);
9081 return DAG.
getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL);
9095 EVT SrcVT = Source.getValueType();
9096 EVT ResVT =
N->getValueType(0);
9105 Source = DAG.
getNode(ISD::VP_SETCC,
DL, SrcVT, Source, AllZero,
9113 DAG.
getNode(ISD::VP_SELECT,
DL, ResVecVT, Source, StepVec,
Splat, EVL);
9114 return DAG.
getNode(ISD::VP_REDUCE_UMIN,
DL, ResVT, ExtEVL,
Select, Mask, EVL);
9118 bool IsNegative)
const {
9120 EVT VT =
N->getValueType(0);
9173 EVT VT =
N->getValueType(0);
9205 EVT VT =
N->getValueType(0);
9212 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
9263 EVT VT =
N->getValueType(0);
9272 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
9281 return DAG.
getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL);
9291 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp2,
9295 Tmp4 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
9296 Tmp2 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
9297 return DAG.
getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
9301 Tmp7 = DAG.
getNode(ISD::VP_AND, dl, VT,
Op,
9305 Tmp6 = DAG.
getNode(ISD::VP_AND, dl, VT,
Op,
9306 DAG.
getConstant(255ULL << 16, dl, VT), Mask, EVL);
9309 Tmp5 = DAG.
getNode(ISD::VP_AND, dl, VT,
Op,
9310 DAG.
getConstant(255ULL << 24, dl, VT), Mask, EVL);
9315 Tmp4 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp4,
9316 DAG.
getConstant(255ULL << 24, dl, VT), Mask, EVL);
9319 Tmp3 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp3,
9320 DAG.
getConstant(255ULL << 16, dl, VT), Mask, EVL);
9323 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp2,
9327 Tmp8 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL);
9328 Tmp6 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL);
9329 Tmp4 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
9330 Tmp2 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
9331 Tmp8 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL);
9332 Tmp4 = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
9333 return DAG.
getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL);
9339 EVT VT =
N->getValueType(0);
9382 for (
unsigned I = 0, J = Sz-1;
I < Sz; ++
I, --J) {
9399 assert(
N->getOpcode() == ISD::VP_BITREVERSE);
9402 EVT VT =
N->getValueType(0);
9421 Tmp = (Sz > 8 ? DAG.
getNode(ISD::VP_BSWAP, dl, VT,
Op, Mask, EVL) :
Op);
9426 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp2,
9432 Tmp = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9437 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp2,
9443 Tmp = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9448 Tmp2 = DAG.
getNode(ISD::VP_AND, dl, VT, Tmp2,
9454 Tmp = DAG.
getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9460std::pair<SDValue, SDValue>
9464 SDValue Chain = LD->getChain();
9465 SDValue BasePTR = LD->getBasePtr();
9466 EVT SrcVT = LD->getMemoryVT();
9467 EVT DstVT = LD->getValueType(0);
9499 LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
9500 LD->getMemOperand()->getFlags(), LD->getAAInfo());
9503 for (
unsigned Idx = 0;
Idx < NumElem; ++
Idx) {
9504 unsigned ShiftIntoIdx =
9516 Scalar = DAG.
getNode(ExtendOp, SL, DstEltVT, Scalar);
9523 return std::make_pair(
Value, Load.getValue(1));
9532 for (
unsigned Idx = 0;
Idx < NumElem; ++
Idx) {
9534 DAG.
getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
9535 LD->getPointerInfo().getWithOffset(
Idx * Stride),
9536 SrcEltVT, LD->getOriginalAlign(),
9537 LD->getMemOperand()->getFlags(), LD->getAAInfo());
9548 return std::make_pair(
Value, NewChain);
9555 SDValue Chain = ST->getChain();
9556 SDValue BasePtr = ST->getBasePtr();
9558 EVT StVT = ST->getMemoryVT();
9584 for (
unsigned Idx = 0;
Idx < NumElem; ++
Idx) {
9589 unsigned ShiftIntoIdx =
9598 return DAG.
getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
9599 ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
9605 assert(Stride &&
"Zero stride!");
9609 for (
unsigned Idx = 0;
Idx < NumElem; ++
Idx) {
9618 Chain, SL, Elt,
Ptr, ST->getPointerInfo().getWithOffset(
Idx * Stride),
9619 MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
9628std::pair<SDValue, SDValue>
9631 "unaligned indexed loads not implemented!");
9632 SDValue Chain = LD->getChain();
9634 EVT VT = LD->getValueType(0);
9635 EVT LoadedVT = LD->getMemoryVT();
9645 return scalarizeVectorLoad(LD, DAG);
9651 LD->getMemOperand());
9657 return std::make_pair(Result, newLoad.
getValue(1));
9665 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
9669 auto FrameIndex = cast<FrameIndexSDNode>(StackBase.
getNode())->getIndex();
9674 EVT PtrVT =
Ptr.getValueType();
9675 EVT StackPtrVT = StackPtr.getValueType();
9681 for (
unsigned i = 1; i < NumRegs; i++) {
9684 RegVT, dl, Chain,
Ptr, LD->getPointerInfo().getWithOffset(
Offset),
9685 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
9689 Load.getValue(1), dl, Load, StackPtr,
9700 8 * (LoadedBytes -
Offset));
9703 LD->getPointerInfo().getWithOffset(
Offset), MemVT,
9704 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
9710 Load.getValue(1), dl, Load, StackPtr,
9717 Load = DAG.
getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
9722 return std::make_pair(Load, TF);
9726 "Unaligned load of unsupported type.");
9735 Align Alignment = LD->getOriginalAlign();
9736 unsigned IncrementSize = NumBits / 8;
9747 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9752 LD->getPointerInfo().getWithOffset(IncrementSize),
9753 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9756 Hi = DAG.
getExtLoad(HiExtType, dl, VT, Chain,
Ptr, LD->getPointerInfo(),
9757 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9762 LD->getPointerInfo().getWithOffset(IncrementSize),
9763 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9775 return std::make_pair(Result, TF);
9781 "unaligned indexed stores not implemented!");
9782 SDValue Chain = ST->getChain();
9786 Align Alignment = ST->getOriginalAlign();
9788 EVT StoreMemVT = ST->getMemoryVT();
9804 Result = DAG.
getStore(Chain, dl, Result,
Ptr, ST->getPointerInfo(),
9805 Alignment, ST->getMemOperand()->getFlags());
9813 EVT PtrVT =
Ptr.getValueType();
9816 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
9820 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
9824 Chain, dl, Val, StackPtr,
9827 EVT StackPtrVT = StackPtr.getValueType();
9835 for (
unsigned i = 1; i < NumRegs; i++) {
9838 RegVT, dl, Store, StackPtr,
9842 ST->getPointerInfo().getWithOffset(
Offset),
9843 ST->getOriginalAlign(),
9844 ST->getMemOperand()->getFlags()));
9864 ST->getPointerInfo().getWithOffset(
Offset), LoadMemVT,
9865 ST->getOriginalAlign(),
9866 ST->getMemOperand()->getFlags(), ST->getAAInfo()));
9873 "Unaligned store of unknown type.");
9877 unsigned IncrementSize = NumBits / 8;
9886 if (
auto *
C = dyn_cast<ConstantSDNode>(
Lo);
C && !
C->isOpaque())
9897 Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
9898 ST->getMemOperand()->getFlags());
9903 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
9904 ST->getMemOperand()->getFlags(), ST->getAAInfo());
9915 bool IsCompressedMemory)
const {
9917 EVT AddrVT =
Addr.getValueType();
9918 EVT MaskVT = Mask.getValueType();
9920 "Incompatible types of Data and Mask");
9921 if (IsCompressedMemory) {
9924 "Cannot currently handle compressed memory with scalable vectors");
9930 MaskIntVT = MVT::i32;
9954 "Cannot index a scalable vector within a fixed-width vector");
9958 EVT IdxVT =
Idx.getValueType();
9964 if (
auto *IdxCst = dyn_cast<ConstantSDNode>(
Idx))
9965 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
9979 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
9987 return getVectorSubVecPointer(
10006 "Converting bits to bytes lost precision");
10008 "Sub-vector must be a vector with matching element type");
10037 std::string NameString = (
"__emutls_v." + GA->
getGlobal()->
getName()).str();
10041 assert(EmuTlsVar &&
"Cannot find EmuTlsVar ");
10043 Entry.Ty = VoidPtrType;
10044 Args.push_back(Entry);
10051 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
10060 "Emulated TLS must have zero offset in GlobalAddressSDNode");
10061 return CallResult.first;
10072 EVT VT =
Op.getOperand(0).getValueType();
10074 if (VT.
bitsLT(MVT::i32)) {
10088 SDValue Op0 = Node->getOperand(0);
10089 SDValue Op1 = Node->getOperand(1);
10092 unsigned Opcode = Node->getOpcode();
10134 {Op0, Op1, DAG.getCondCode(CC)})) {
10141 {Op0, Op1, DAG.getCondCode(CC)})) {
10169 unsigned Opcode = Node->getOpcode();
10172 EVT VT =
LHS.getValueType();
10175 assert(VT ==
RHS.getValueType() &&
"Expected operands to be the same type");
10191 unsigned OverflowOp;
10206 llvm_unreachable(
"Expected method to receive signed or unsigned saturation "
10207 "addition or subtraction node.");
10215 unsigned BitWidth =
LHS.getScalarValueSizeInBits();
10218 SDValue SumDiff = Result.getValue(0);
10219 SDValue Overflow = Result.getValue(1);
10241 return DAG.
getSelect(dl, VT, Overflow, Zero, SumDiff);
10261 if (LHSIsNonNegative || RHSIsNonNegative) {
10263 return DAG.
getSelect(dl, VT, Overflow, SatMax, SumDiff);
10269 if (LHSIsNegative || RHSIsNegative) {
10271 return DAG.
getSelect(dl, VT, Overflow, SatMin, SumDiff);
10281 return DAG.
getSelect(dl, VT, Overflow, Result, SumDiff);
10285 unsigned Opcode = Node->getOpcode();
10289 EVT VT =
LHS.getValueType();
10294 "Expected a SHLSAT opcode");
10295 assert(VT ==
RHS.getValueType() &&
"Expected operands to be the same type");
10333 if (WideVT == MVT::i16)
10334 LC = RTLIB::MUL_I16;
10335 else if (WideVT == MVT::i32)
10336 LC = RTLIB::MUL_I32;
10337 else if (WideVT == MVT::i64)
10338 LC = RTLIB::MUL_I64;
10339 else if (WideVT == MVT::i128)
10340 LC = RTLIB::MUL_I128;
10349 unsigned HalfBits = Bits >> 1;
10388 if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.
getDataLayout())) {
10393 SDValue Args[] = {LL, LH, RL, RH};
10394 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
10396 SDValue Args[] = {LH, LL, RH, RL};
10397 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
10400 "Ret value is a collection of constituent nodes holding result.");
10403 Lo = Ret.getOperand(0);
10404 Hi = Ret.getOperand(1);
10406 Lo = Ret.getOperand(1);
10407 Hi = Ret.getOperand(0);
10416 EVT VT =
LHS.getValueType();
10417 assert(
RHS.getValueType() == VT &&
"Mismatching operand types");
10436 forceExpandWideMUL(DAG, dl,
Signed, WideVT,
LHS, HiLHS,
RHS, HiRHS,
Lo,
Hi);
10445 "Expected a fixed point multiplication opcode");
10450 EVT VT =
LHS.getValueType();
10451 unsigned Scale = Node->getConstantOperandVal(2);
10467 SDValue Product = Result.getValue(0);
10468 SDValue Overflow = Result.getValue(1);
10479 Result = DAG.
getSelect(dl, VT, ProdNeg, SatMin, SatMax);
10480 return DAG.
getSelect(dl, VT, Overflow, Result, Product);
10484 SDValue Product = Result.getValue(0);
10485 SDValue Overflow = Result.getValue(1);
10489 return DAG.
getSelect(dl, VT, Overflow, SatMax, Product);
10494 "Expected scale to be less than the number of bits if signed or at "
10495 "most the number of bits if unsigned.");
10497 "Expected both operands to be the same type");
10505 Lo = Result.getValue(0);
10506 Hi = Result.getValue(1);
10516 if (Scale == VTSize)
10562 return DAG.
getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
10587 "Expected a fixed point division opcode");
10589 EVT VT =
LHS.getValueType();
10611 if (LHSLead + RHSTrail < Scale + (
unsigned)(Saturating &&
Signed))
10614 unsigned LHSShift = std::min(LHSLead, Scale);
10615 unsigned RHSShift = Scale - LHSShift;
10672 bool IsAdd = Node->getOpcode() ==
ISD::UADDO;
10678 SDValue NodeCarry = DAG.
getNode(OpcCarry, dl, Node->getVTList(),
10679 { LHS, RHS, CarryIn });
10688 EVT ResultType = Node->getValueType(1);
10699 DAG.
getSetCC(dl, SetCCType, Result,
10718 bool IsAdd = Node->getOpcode() ==
ISD::SADDO;
10723 EVT ResultType = Node->getValueType(1);
10749 DAG.
getNode(
ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
10750 ResultType, ResultType);
10756 EVT VT = Node->getValueType(0);
10764 const APInt &
C = RHSC->getAPIntValue();
10766 if (
C.isPowerOf2()) {
10768 bool UseArithShift =
isSigned && !
C.isMinSignedValue();
10771 Overflow = DAG.
getSetCC(dl, SetCCVT,
10773 dl, VT, Result, ShiftAmt),
10786 static const unsigned Ops[2][3] =
10809 forceExpandWideMUL(DAG, dl,
isSigned,
LHS,
RHS, BottomHalf, TopHalf);
10812 Result = BottomHalf;
10819 Overflow = DAG.
getSetCC(dl, SetCCVT, TopHalf,
10824 EVT RType = Node->getValueType(1);
10829 "Unexpected result type for S/UMULO legalization");
10837 EVT VT =
Op.getValueType();
10841 "Expanding reductions for scalable vectors is undefined.");
10852 Op = DAG.
getNode(BaseOpcode, dl, HalfVT,
Lo,
Hi, Node->getFlags());
10864 for (
unsigned i = 1; i < NumElts; i++)
10865 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());
10868 if (EltVT != Node->getValueType(0))
10875 SDValue AccOp = Node->getOperand(0);
10876 SDValue VecOp = Node->getOperand(1);
10884 "Expanding reductions for scalable vectors is undefined.");
10894 for (
unsigned i = 0; i < NumElts; i++)
10895 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
10902 EVT VT = Node->getValueType(0);
10907 SDValue Dividend = Node->getOperand(0);
10908 SDValue Divisor = Node->getOperand(1);
10911 Result = DAG.
getNode(DivRemOpc, dl, VTs, Dividend, Divisor).
getValue(1);
10916 SDValue Divide = DAG.
getNode(DivOpc, dl, VT, Dividend, Divisor);
10928 SDValue Src = Node->getOperand(0);
10931 EVT SrcVT = Src.getValueType();
10932 EVT DstVT = Node->getValueType(0);
10934 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
10937 assert(SatWidth <= DstWidth &&
10938 "Expected saturation width smaller than result width");
10942 APInt MinInt, MaxInt;
10953 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
10955 SrcVT = Src.getValueType();
10976 if (AreExactFloatBounds && MinMaxLegal) {
10985 dl, DstVT, Clamped);
10997 return DAG.
getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
11036 EVT OperandVT =
Op.getValueType();
11058 AbsWide = DAG.
getBitcast(OperandVT, ClearedSign);
11081 KeepNarrow = DAG.
getNode(
ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
11090 SDValue Adjust = DAG.
getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
11092 Op = DAG.
getSelect(dl, ResultIntVT, KeepNarrow, NarrowBits, Adjusted);
11104 EVT VT = Node->getValueType(0);
11107 if (Node->getConstantOperandVal(1) == 1) {
11110 EVT OperandVT =
Op.getValueType();
11122 EVT I32 = F32.changeTypeToInteger();
11123 Op = expandRoundInexactToOdd(F32,
Op, dl, DAG);
11148 EVT I16 = I32.isVector() ? I32.changeVectorElementType(MVT::i16) : MVT::i16;
11158 assert(Node->getValueType(0).isScalableVector() &&
11159 "Fixed length vector types expected to use SHUFFLE_VECTOR!");
11161 EVT VT = Node->getValueType(0);
11162 SDValue V1 = Node->getOperand(0);
11163 SDValue V2 = Node->getOperand(1);
11164 int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
11183 EVT PtrVT = StackPtr.getValueType();
11185 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
11200 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
11202 return DAG.
getLoad(VT,
DL, StoreV2, StackPtr,
11225 return DAG.
getLoad(VT,
DL, StoreV2, StackPtr2,
11232 SDValue EVL,
bool &NeedInvert,
11234 bool IsSignaling)
const {
11236 MVT OpVT =
LHS.getSimpleValueType();
11238 NeedInvert =
false;
11239 assert(!EVL == !Mask &&
"VP Mask and EVL must either both be set or unset");
11240 bool IsNonVP = !EVL;
11255 bool NeedSwap =
false;
11256 InvCC = getSetCCInverse(CCCode, OpVT);
11284 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
11289 "If SETO is expanded, SETOEQ must be legal!");
11307 NeedInvert = ((
unsigned)CCCode & 0x8U);
11348 SetCC1 = DAG.
getSetCC(dl, VT,
LHS,
RHS, CC1, Chain, IsSignaling);
11349 SetCC2 = DAG.
getSetCC(dl, VT,
LHS,
RHS, CC2, Chain, IsSignaling);
11357 SetCC1 = DAG.
getSetCC(dl, VT,
LHS,
LHS, CC1, Chain, IsSignaling);
11358 SetCC2 = DAG.
getSetCC(dl, VT,
RHS,
RHS, CC2, Chain, IsSignaling);
11368 LHS = DAG.
getNode(Opc, dl, VT, SetCC1, SetCC2);
11372 Opc = Opc ==
ISD::OR ? ISD::VP_OR : ISD::VP_AND;
11373 LHS = DAG.
getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL);
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
static bool isUndef(ArrayRef< int > Mask)
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI)
If we have an immediate, see if we can lower it.
static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG)
static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1)
Given a vector binary operation and known undefined elements for each input operand,...
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl, ElementCount SubEC)
static unsigned getConstraintPiority(TargetLowering::ConstraintType CT)
Return a number indicating our preference for chosing a type of constraint over another,...
static std::optional< bool > isFCmpEqualZero(FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF)
Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0,...
static void turnVectorIntoSplatVector(MutableArrayRef< SDValue > Values, std::function< bool(SDValue)> Predicate, SDValue AlternativeReplacement=SDValue())
If all values in Values that don't match the predicate are same 'splat' value, then replace all value...
static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT)
static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
This file describes how to lower LLVM code to machine code.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
APInt udiv(const APInt &RHS) const
Unsigned division operation.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
void setSignBit()
Set the sign bit to 1.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
void clearAllBits()
Set every bit to 0.
APInt reverseBits() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
void negate()
Negate this APInt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned countLeadingZeros() const
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
void setAllBits()
Set every bit to 1.
APInt multiplicativeInverse() const
bool isMaxSignedValue() const
Determine if this is the largest signed value.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
void setBitVal(unsigned BitPosition, bool BitValue)
Set a given bit to a given value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool hasAttributes() const
Return true if the builder has IR-level attributes.
bool contains(Attribute::AttrKind A) const
Return true if the builder has the specified attribute.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
ConstantFP - Floating Point Values [float, double].
const APInt & getAPIntValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
AttributeList getAttributes() const
Return the attribute list for this Function.
int64_t getOffset() const
const GlobalValue * getGlobal() const
Module * getParent()
Get the module that this global value is contained inside of...
std::vector< std::string > ConstraintCodeVector
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
Wrapper class representing physical registers. Should be passed by value.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setAdjustsStack(bool V)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MCSymbol * getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate=false) const
getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
Function & getFunction()
Return the LLVM function that this machine code represents.
@ EK_GPRel32BlockAddress
EK_GPRel32BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative,...
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
@ EK_GPRel64BlockAddress
EK_GPRel64BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative,...
Flags getFlags() const
Return the raw flags of the source value,.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
unsigned getAddressSpace() const
Return the address space for the associated pointer.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Class to represent pointers.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static SDNodeIterator end(const SDNode *N)
static SDNodeIterator begin(const SDNode *N)
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const APInt * getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
const APInt * getValidShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has a constant or splat constant shift amount that is less than the element b...
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
size - Get the string size.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Class to represent struct types.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
const TargetMachine & getTargetMachine() const
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const
Indicates if any padding is guaranteed to go at the most significant bits when storing the type to me...
virtual EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const
Get the CondCode that's to be used to test the result of the comparison libcall against zero.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom on this target.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
SDValue expandVPCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTLZ/VP_CTLZ_ZERO_UNDEF nodes.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
virtual const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const
virtual unsigned computeNumSignBitsForTargetInstr(GISelKnownBits &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand VP_BSWAP nodes.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
virtual Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine the known alignment for the pointer value R.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all bits from only some vector eleme...
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand VP_BITREVERSE nodes.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_UNDEF nodes.
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimum/fmaximum into multiple comparison with selects.
SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned NumBitsPerElt) const
Expand CTTZ via Table Lookup.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, SDValue Mask, SDValue EVL, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC or VP_SETCC with given LHS and RHS and condition code CC on the current target.
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const
Return if N is a True value when extended to VT.
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &DemandedBits, TargetLoweringOpt &TLO) const
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue expandVPCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTPOP nodes.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
SDValue expandVPCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ/VP_CTTZ_ZERO_UNDEF nodes.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][MIN|MAX].
virtual void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
Primary interface to the complete machine description for the target machine.
bool isPositionIndependent() const
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
iterator_range< regclass_iterator > regclasses() const
virtual StringRef getRegAsmName(MCRegister Reg) const
Return the assembly name for Reg.
bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const
Return true if the given TargetRegisterClass has the ValueType T.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
const fltSemantics & getFltSemantics() const
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isIntegerTy() const
True if this is an instance of IntegerType.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
FPClassTest invertFPClassTestIfSimpler(FPClassTest Test)
Evaluates if the specified FP class test is better performed as the inverse (i.e.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
auto find_if_not(R &&Range, UnaryPredicate P)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isConstFalseVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
constexpr unsigned BitWidth
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
APFloat neg(APFloat X)
Returns the negated value of the argument.
unsigned Log2(Align A)
Returns the log2 of the alignment.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static constexpr roundingMode rmTowardZero
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Represent subnormal handling kind for floating point instruction inputs and outputs.
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
@ IEEE
IEEE-754 denormal numbers preserved.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
int MatchingInput
MatchingInput - If this is not -1, this is an output constraint where an input constraint is required...
ConstraintCodeVector Codes
Code - The constraint code, either the register name (in braces) or the constraint letter/number.
SubConstraintInfoVector multipleAlternatives
multipleAlternatives - If there are multiple alternative constraints, this array will contain them.
bool isIndirect
isIndirect - True if this operand is an indirect operand.
bool hasMatchingInput() const
hasMatchingInput - Return true if this is an output constraint that has a matching input constraint.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
bool isUnknown() const
Returns true if we don't know any bits.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
bool hasConflict() const
Returns true if there is conflicting information.
static std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
unsigned getBitWidth() const
Get the bit width of this value.
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
bool isNegative() const
Returns true if this value is known to be negative.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
static std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
unsigned countMinPopulation() const
Returns the number of bits known to be one.
static std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Magic data for optimising signed division by a constant.
unsigned ShiftAmount
shift amount
static SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This contains information for each constraint that we are lowering.
MVT ConstraintVT
The ValueType for the operand value.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
std::string ConstraintCode
This contains the actual string for the code, like "m".
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
unsigned getMatchedOperand() const
If this is an input matching constraint, this method returns the output operand it matches.
bool isMatchingInputConstraint() const
Return true of this is an input operand that is a matching constraint like "4".
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)
bool isBeforeLegalizeOps() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
ArrayRef< EVT > OpsVTBeforeSoften
bool IsPostTypeLegalization
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)
bool LegalOperations() const
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
static UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
unsigned PostShift
post-shift amount