38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
551 if (Subtarget.hasStdExtZfa()) {
568 if (Subtarget.hasStdExtZfa()) {
645 if (Subtarget.hasStdExtZicbop()) {
649 if (Subtarget.hasStdExtA()) {
651 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
655 }
else if (Subtarget.hasForcedAtomics()) {
676 {MVT::i8, MVT::i16},
Custom);
687 static const unsigned IntegerVPOps[] = {
688 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
689 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
690 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
691 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
692 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
693 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
694 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
695 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
696 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
697 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
698 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
699 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
700 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
701 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF};
703 static const unsigned FloatingPointVPOps[] = {
704 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
705 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
706 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
707 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
708 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
709 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
710 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
711 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
712 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
713 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
714 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
715 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
716 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
717 ISD::VP_REDUCE_FMAXIMUM};
719 static const unsigned IntegerVecReduceOps[] = {
724 static const unsigned FloatingPointVecReduceOps[] = {
737 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
738 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
739 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
743 for (
MVT VT : BoolVecVTs) {
773 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
797 ISD::VP_TRUNCATE, ISD::VP_SETCC},
813 for (
MVT VT : IntVecVTs) {
824 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
871 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
872 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
896 if (Subtarget.hasStdExtZvkb()) {
904 if (Subtarget.hasStdExtZvbb()) {
908 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
914 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
923 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
943 static const unsigned ZvfhminPromoteOps[] = {
953 static const unsigned ZvfhminPromoteVPOps[] = {
954 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
955 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
956 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
957 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
958 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
959 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
960 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
961 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
962 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
965 const auto SetCommonVFPActions = [&](
MVT VT) {
1010 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1011 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1042 const auto SetCommonVFPExtLoadTruncStoreActions =
1044 for (
auto SmallVT : SmallerVTs) {
1051 for (
MVT VT : F16VecVTs) {
1054 SetCommonVFPActions(VT);
1057 for (
MVT VT : F16VecVTs) {
1068 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1079 if (VT == MVT::nxv32f16) {
1093 for (
MVT VT : BF16VecVTs) {
1109 for (
MVT VT : F32VecVTs) {
1112 SetCommonVFPActions(VT);
1113 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1118 for (
MVT VT : F64VecVTs) {
1121 SetCommonVFPActions(VT);
1122 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1123 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1129 if (!useRVVForFixedLengthVectorVT(VT))
1172 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1199 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1221 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1222 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1259 if (Subtarget.hasStdExtZvkb())
1262 if (Subtarget.hasStdExtZvbb()) {
1284 if (!useRVVForFixedLengthVectorVT(VT))
1305 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1350 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1351 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1402 if (Subtarget.hasStdExtA()) {
1408 if (Subtarget.hasForcedAtomics()) {
1418 if (Subtarget.hasVendorXTHeadMemIdx()) {
1450 if (Subtarget.hasStdExtZbb())
1453 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1456 if (Subtarget.hasStdExtZbkb())
1468 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1471 if (Subtarget.hasVendorXTHeadMemPair())
1494MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1499bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1501 bool IsScalable)
const {
1508 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1525bool RISCVTargetLowering::shouldExpandCttzElements(
EVT VT)
const {
1533 unsigned Intrinsic)
const {
1534 auto &
DL =
I.getModule()->getDataLayout();
1536 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1537 bool IsUnitStrided,
bool UsePtrVal =
false) {
1542 Info.ptrVal =
I.getArgOperand(PtrOp);
1544 Info.fallbackAddressSpace =
1545 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1549 MemTy =
I.getArgOperand(0)->getType();
1552 MemTy =
I.getType();
1567 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1571 switch (Intrinsic) {
1574 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1575 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1576 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1577 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1578 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1579 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1580 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1581 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1582 case Intrinsic::riscv_masked_cmpxchg_i32:
1584 Info.memVT = MVT::i32;
1585 Info.ptrVal =
I.getArgOperand(0);
1591 case Intrinsic::riscv_masked_strided_load:
1592 return SetRVVLoadStoreInfo( 1,
false,
1594 case Intrinsic::riscv_masked_strided_store:
1595 return SetRVVLoadStoreInfo( 1,
true,
1597 case Intrinsic::riscv_seg2_load:
1598 case Intrinsic::riscv_seg3_load:
1599 case Intrinsic::riscv_seg4_load:
1600 case Intrinsic::riscv_seg5_load:
1601 case Intrinsic::riscv_seg6_load:
1602 case Intrinsic::riscv_seg7_load:
1603 case Intrinsic::riscv_seg8_load:
1604 return SetRVVLoadStoreInfo( 0,
false,
1606 case Intrinsic::riscv_seg2_store:
1607 case Intrinsic::riscv_seg3_store:
1608 case Intrinsic::riscv_seg4_store:
1609 case Intrinsic::riscv_seg5_store:
1610 case Intrinsic::riscv_seg6_store:
1611 case Intrinsic::riscv_seg7_store:
1612 case Intrinsic::riscv_seg8_store:
1614 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1617 case Intrinsic::riscv_vle:
1618 case Intrinsic::riscv_vle_mask:
1619 case Intrinsic::riscv_vleff:
1620 case Intrinsic::riscv_vleff_mask:
1621 return SetRVVLoadStoreInfo( 1,
1625 case Intrinsic::riscv_vse:
1626 case Intrinsic::riscv_vse_mask:
1627 return SetRVVLoadStoreInfo( 1,
1631 case Intrinsic::riscv_vlse:
1632 case Intrinsic::riscv_vlse_mask:
1633 case Intrinsic::riscv_vloxei:
1634 case Intrinsic::riscv_vloxei_mask:
1635 case Intrinsic::riscv_vluxei:
1636 case Intrinsic::riscv_vluxei_mask:
1637 return SetRVVLoadStoreInfo( 1,
1640 case Intrinsic::riscv_vsse:
1641 case Intrinsic::riscv_vsse_mask:
1642 case Intrinsic::riscv_vsoxei:
1643 case Intrinsic::riscv_vsoxei_mask:
1644 case Intrinsic::riscv_vsuxei:
1645 case Intrinsic::riscv_vsuxei_mask:
1646 return SetRVVLoadStoreInfo( 1,
1649 case Intrinsic::riscv_vlseg2:
1650 case Intrinsic::riscv_vlseg3:
1651 case Intrinsic::riscv_vlseg4:
1652 case Intrinsic::riscv_vlseg5:
1653 case Intrinsic::riscv_vlseg6:
1654 case Intrinsic::riscv_vlseg7:
1655 case Intrinsic::riscv_vlseg8:
1656 case Intrinsic::riscv_vlseg2ff:
1657 case Intrinsic::riscv_vlseg3ff:
1658 case Intrinsic::riscv_vlseg4ff:
1659 case Intrinsic::riscv_vlseg5ff:
1660 case Intrinsic::riscv_vlseg6ff:
1661 case Intrinsic::riscv_vlseg7ff:
1662 case Intrinsic::riscv_vlseg8ff:
1663 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1666 case Intrinsic::riscv_vlseg2_mask:
1667 case Intrinsic::riscv_vlseg3_mask:
1668 case Intrinsic::riscv_vlseg4_mask:
1669 case Intrinsic::riscv_vlseg5_mask:
1670 case Intrinsic::riscv_vlseg6_mask:
1671 case Intrinsic::riscv_vlseg7_mask:
1672 case Intrinsic::riscv_vlseg8_mask:
1673 case Intrinsic::riscv_vlseg2ff_mask:
1674 case Intrinsic::riscv_vlseg3ff_mask:
1675 case Intrinsic::riscv_vlseg4ff_mask:
1676 case Intrinsic::riscv_vlseg5ff_mask:
1677 case Intrinsic::riscv_vlseg6ff_mask:
1678 case Intrinsic::riscv_vlseg7ff_mask:
1679 case Intrinsic::riscv_vlseg8ff_mask:
1680 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1683 case Intrinsic::riscv_vlsseg2:
1684 case Intrinsic::riscv_vlsseg3:
1685 case Intrinsic::riscv_vlsseg4:
1686 case Intrinsic::riscv_vlsseg5:
1687 case Intrinsic::riscv_vlsseg6:
1688 case Intrinsic::riscv_vlsseg7:
1689 case Intrinsic::riscv_vlsseg8:
1690 case Intrinsic::riscv_vloxseg2:
1691 case Intrinsic::riscv_vloxseg3:
1692 case Intrinsic::riscv_vloxseg4:
1693 case Intrinsic::riscv_vloxseg5:
1694 case Intrinsic::riscv_vloxseg6:
1695 case Intrinsic::riscv_vloxseg7:
1696 case Intrinsic::riscv_vloxseg8:
1697 case Intrinsic::riscv_vluxseg2:
1698 case Intrinsic::riscv_vluxseg3:
1699 case Intrinsic::riscv_vluxseg4:
1700 case Intrinsic::riscv_vluxseg5:
1701 case Intrinsic::riscv_vluxseg6:
1702 case Intrinsic::riscv_vluxseg7:
1703 case Intrinsic::riscv_vluxseg8:
1704 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1707 case Intrinsic::riscv_vlsseg2_mask:
1708 case Intrinsic::riscv_vlsseg3_mask:
1709 case Intrinsic::riscv_vlsseg4_mask:
1710 case Intrinsic::riscv_vlsseg5_mask:
1711 case Intrinsic::riscv_vlsseg6_mask:
1712 case Intrinsic::riscv_vlsseg7_mask:
1713 case Intrinsic::riscv_vlsseg8_mask:
1714 case Intrinsic::riscv_vloxseg2_mask:
1715 case Intrinsic::riscv_vloxseg3_mask:
1716 case Intrinsic::riscv_vloxseg4_mask:
1717 case Intrinsic::riscv_vloxseg5_mask:
1718 case Intrinsic::riscv_vloxseg6_mask:
1719 case Intrinsic::riscv_vloxseg7_mask:
1720 case Intrinsic::riscv_vloxseg8_mask:
1721 case Intrinsic::riscv_vluxseg2_mask:
1722 case Intrinsic::riscv_vluxseg3_mask:
1723 case Intrinsic::riscv_vluxseg4_mask:
1724 case Intrinsic::riscv_vluxseg5_mask:
1725 case Intrinsic::riscv_vluxseg6_mask:
1726 case Intrinsic::riscv_vluxseg7_mask:
1727 case Intrinsic::riscv_vluxseg8_mask:
1728 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1731 case Intrinsic::riscv_vsseg2:
1732 case Intrinsic::riscv_vsseg3:
1733 case Intrinsic::riscv_vsseg4:
1734 case Intrinsic::riscv_vsseg5:
1735 case Intrinsic::riscv_vsseg6:
1736 case Intrinsic::riscv_vsseg7:
1737 case Intrinsic::riscv_vsseg8:
1738 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1741 case Intrinsic::riscv_vsseg2_mask:
1742 case Intrinsic::riscv_vsseg3_mask:
1743 case Intrinsic::riscv_vsseg4_mask:
1744 case Intrinsic::riscv_vsseg5_mask:
1745 case Intrinsic::riscv_vsseg6_mask:
1746 case Intrinsic::riscv_vsseg7_mask:
1747 case Intrinsic::riscv_vsseg8_mask:
1748 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1751 case Intrinsic::riscv_vssseg2:
1752 case Intrinsic::riscv_vssseg3:
1753 case Intrinsic::riscv_vssseg4:
1754 case Intrinsic::riscv_vssseg5:
1755 case Intrinsic::riscv_vssseg6:
1756 case Intrinsic::riscv_vssseg7:
1757 case Intrinsic::riscv_vssseg8:
1758 case Intrinsic::riscv_vsoxseg2:
1759 case Intrinsic::riscv_vsoxseg3:
1760 case Intrinsic::riscv_vsoxseg4:
1761 case Intrinsic::riscv_vsoxseg5:
1762 case Intrinsic::riscv_vsoxseg6:
1763 case Intrinsic::riscv_vsoxseg7:
1764 case Intrinsic::riscv_vsoxseg8:
1765 case Intrinsic::riscv_vsuxseg2:
1766 case Intrinsic::riscv_vsuxseg3:
1767 case Intrinsic::riscv_vsuxseg4:
1768 case Intrinsic::riscv_vsuxseg5:
1769 case Intrinsic::riscv_vsuxseg6:
1770 case Intrinsic::riscv_vsuxseg7:
1771 case Intrinsic::riscv_vsuxseg8:
1772 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1775 case Intrinsic::riscv_vssseg2_mask:
1776 case Intrinsic::riscv_vssseg3_mask:
1777 case Intrinsic::riscv_vssseg4_mask:
1778 case Intrinsic::riscv_vssseg5_mask:
1779 case Intrinsic::riscv_vssseg6_mask:
1780 case Intrinsic::riscv_vssseg7_mask:
1781 case Intrinsic::riscv_vssseg8_mask:
1782 case Intrinsic::riscv_vsoxseg2_mask:
1783 case Intrinsic::riscv_vsoxseg3_mask:
1784 case Intrinsic::riscv_vsoxseg4_mask:
1785 case Intrinsic::riscv_vsoxseg5_mask:
1786 case Intrinsic::riscv_vsoxseg6_mask:
1787 case Intrinsic::riscv_vsoxseg7_mask:
1788 case Intrinsic::riscv_vsoxseg8_mask:
1789 case Intrinsic::riscv_vsuxseg2_mask:
1790 case Intrinsic::riscv_vsuxseg3_mask:
1791 case Intrinsic::riscv_vsuxseg4_mask:
1792 case Intrinsic::riscv_vsuxseg5_mask:
1793 case Intrinsic::riscv_vsuxseg6_mask:
1794 case Intrinsic::riscv_vsuxseg7_mask:
1795 case Intrinsic::riscv_vsuxseg8_mask:
1796 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1833 return isInt<12>(Imm);
1837 return isInt<12>(Imm);
1850 return (SrcBits == 64 && DestBits == 32);
1861 return (SrcBits == 64 && DestBits == 32);
1868 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1869 EVT MemVT = LD->getMemoryVT();
1870 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1880 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1888 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1892 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1893 Subtarget.hasVendorXCVbitmanip();
1904 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1909 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1913 EVT VT =
Y.getValueType();
1919 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1920 !isa<ConstantSDNode>(
Y);
1925 if (Subtarget.hasStdExtZbs())
1926 return X.getValueType().isScalarInteger();
1927 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1929 if (Subtarget.hasVendorXTHeadBs())
1930 return C !=
nullptr;
1932 return C &&
C->getAPIntValue().ule(10);
1952 if (BitSize > Subtarget.
getXLen())
1956 int64_t Val = Imm.getSExtValue();
1964 if (!Subtarget.enableUnalignedScalarMem())
1980 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1987 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1991 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2003 case Instruction::Add:
2004 case Instruction::Sub:
2005 case Instruction::Mul:
2006 case Instruction::And:
2007 case Instruction::Or:
2008 case Instruction::Xor:
2009 case Instruction::FAdd:
2010 case Instruction::FSub:
2011 case Instruction::FMul:
2012 case Instruction::FDiv:
2013 case Instruction::ICmp:
2014 case Instruction::FCmp:
2016 case Instruction::Shl:
2017 case Instruction::LShr:
2018 case Instruction::AShr:
2019 case Instruction::UDiv:
2020 case Instruction::SDiv:
2021 case Instruction::URem:
2022 case Instruction::SRem:
2023 case Instruction::Select:
2024 return Operand == 1;
2038 auto *II = dyn_cast<IntrinsicInst>(
I);
2042 switch (II->getIntrinsicID()) {
2043 case Intrinsic::fma:
2044 case Intrinsic::vp_fma:
2045 return Operand == 0 || Operand == 1;
2046 case Intrinsic::vp_shl:
2047 case Intrinsic::vp_lshr:
2048 case Intrinsic::vp_ashr:
2049 case Intrinsic::vp_udiv:
2050 case Intrinsic::vp_sdiv:
2051 case Intrinsic::vp_urem:
2052 case Intrinsic::vp_srem:
2053 case Intrinsic::ssub_sat:
2054 case Intrinsic::vp_ssub_sat:
2055 case Intrinsic::usub_sat:
2056 case Intrinsic::vp_usub_sat:
2057 return Operand == 1;
2059 case Intrinsic::vp_add:
2060 case Intrinsic::vp_mul:
2061 case Intrinsic::vp_and:
2062 case Intrinsic::vp_or:
2063 case Intrinsic::vp_xor:
2064 case Intrinsic::vp_fadd:
2065 case Intrinsic::vp_fmul:
2066 case Intrinsic::vp_icmp:
2067 case Intrinsic::vp_fcmp:
2068 case Intrinsic::smin:
2069 case Intrinsic::vp_smin:
2070 case Intrinsic::umin:
2071 case Intrinsic::vp_umin:
2072 case Intrinsic::smax:
2073 case Intrinsic::vp_smax:
2074 case Intrinsic::umax:
2075 case Intrinsic::vp_umax:
2076 case Intrinsic::sadd_sat:
2077 case Intrinsic::vp_sadd_sat:
2078 case Intrinsic::uadd_sat:
2079 case Intrinsic::vp_uadd_sat:
2081 case Intrinsic::vp_sub:
2082 case Intrinsic::vp_fsub:
2083 case Intrinsic::vp_fdiv:
2084 return Operand == 0 || Operand == 1;
2105 if (!Subtarget.sinkSplatOperands())
2108 for (
auto OpIdx :
enumerate(
I->operands())) {
2112 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2114 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2123 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2128 for (
Use &U :
Op->uses()) {
2180 if (!Subtarget.hasStdExtZfa())
2181 return std::make_pair(-1,
false);
2183 bool IsSupportedVT =
false;
2184 if (VT == MVT::f16) {
2185 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2186 }
else if (VT == MVT::f32) {
2187 IsSupportedVT =
true;
2188 }
else if (VT == MVT::f64) {
2189 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2190 IsSupportedVT =
true;
2194 return std::make_pair(-1,
false);
2197 if (
Index < 0 && Imm.isNegative())
2201 return std::make_pair(
Index,
false);
2205 bool ForCodeSize)
const {
2206 bool IsLegalVT =
false;
2209 else if (VT == MVT::f32)
2211 else if (VT == MVT::f64)
2213 else if (VT == MVT::bf16)
2214 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2226 return Imm.isZero();
2230 if (Imm.isNegZero())
2243 unsigned Index)
const {
2256 if (EltVT == MVT::i1)
2269 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2276 if ((ResElts * 2) != SrcElts)
2320 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2322 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2325 IntermediateVT = MVT::i64;
2328 RegisterVT = MVT::i64;
2343 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2349 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2362 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2363 int64_t
C = RHSC->getSExtValue();
2405 switch (KnownSize) {
2433 return RISCV::VRRegClassID;
2435 return RISCV::VRM2RegClassID;
2437 return RISCV::VRM4RegClassID;
2439 return RISCV::VRM8RegClassID;
2449 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2450 "Unexpected subreg numbering");
2451 return RISCV::sub_vrm1_0 +
Index;
2454 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2455 "Unexpected subreg numbering");
2456 return RISCV::sub_vrm2_0 +
Index;
2459 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2460 "Unexpected subreg numbering");
2461 return RISCV::sub_vrm4_0 +
Index;
2468 return RISCV::VRRegClassID;
2477std::pair<unsigned, unsigned>
2479 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2481 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2482 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2483 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2484 "Register classes not ordered");
2493 unsigned SubRegIdx = RISCV::NoSubRegister;
2494 for (
const unsigned RCID :
2495 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2496 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2500 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2505 return {SubRegIdx, InsertExtractIdx};
2510bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2539unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2546 "Unexpected opcode");
2548 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2550 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2553 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2627bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2628 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2637 "Expected legal fixed length vector!");
2640 unsigned MaxELen = Subtarget.
getELen();
2674 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2681 "Expected to convert into a scalable vector!");
2682 assert(V.getValueType().isFixedLengthVector() &&
2683 "Expected a fixed length vector operand!");
2693 "Expected to convert into a fixed length vector!");
2694 assert(V.getValueType().isScalableVector() &&
2695 "Expected a scalable vector operand!");
2723 const auto [MinVLMAX, MaxVLMAX] =
2725 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2731static std::pair<SDValue, SDValue>
2740static std::pair<SDValue, SDValue>
2753static std::pair<SDValue, SDValue>
2770std::pair<unsigned, unsigned>
2786 return std::make_pair(MinVLMAX, MaxVLMAX);
2798 EVT VT,
unsigned DefinedValues)
const {
2812 std::tie(LMul, Fractional) =
2815 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2817 Cost = (LMul * DLenFactor);
2862 MVT DstVT =
Op.getSimpleValueType();
2863 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2871 Src.getValueType() == MVT::bf16) {
2878 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2886 Opc,
DL, DstVT, Src,
2900 MVT SrcVT = Src.getSimpleValueType();
2906 if (SatVT != DstEltVT)
2910 if (SrcEltSize > (2 * DstEltSize))
2913 MVT DstContainerVT = DstVT;
2914 MVT SrcContainerVT = SrcVT;
2920 "Expected same element count");
2929 {Src, Src, DAG.getCondCode(ISD::SETNE),
2930 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2934 if (DstEltSize > (2 * SrcEltSize)) {
2948 Res, DAG.
getUNDEF(DstContainerVT), VL);
2960 case ISD::VP_FROUNDEVEN:
2964 case ISD::VP_FROUNDTOZERO:
2968 case ISD::VP_FFLOOR:
2976 case ISD::VP_FROUND:
2992 MVT VT =
Op.getSimpleValueType();
2999 MVT ContainerVT = VT;
3006 if (
Op->isVPOpcode()) {
3007 Mask =
Op.getOperand(1);
3011 VL =
Op.getOperand(2);
3033 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3047 switch (
Op.getOpcode()) {
3053 case ISD::VP_FFLOOR:
3056 case ISD::VP_FROUND:
3057 case ISD::VP_FROUNDEVEN:
3058 case ISD::VP_FROUNDTOZERO: {
3074 case ISD::VP_FNEARBYINT:
3087 Src, Src, Mask, VL);
3102 MVT VT =
Op.getSimpleValueType();
3106 MVT ContainerVT = VT;
3118 MVT MaskVT = Mask.getSimpleValueType();
3121 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3122 DAG.getUNDEF(MaskVT), Mask, VL});
3126 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
3127 Chain = Src.getValue(1);
3143 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3155 switch (
Op.getOpcode()) {
3166 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3172 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3176 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3185 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3186 Truncated, Mask, VL);
3192 Src, Src, Mask, VL);
3202 MVT VT =
Op.getSimpleValueType();
3230 MVT VT =
Op.getSimpleValueType();
3235 MVT ContainerVT = VT;
3257 if (
Merge.isUndef())
3269 if (
Merge.isUndef())
3278 "Unexpected vector MVT");
3306 return std::nullopt;
3321 unsigned EltSizeInBits) {
3324 return std::nullopt;
3325 bool IsInteger =
Op.getValueType().isInteger();
3327 std::optional<unsigned> SeqStepDenom;
3328 std::optional<int64_t> SeqStepNum, SeqAddend;
3329 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3330 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3335 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3337 if (Elt.isUndef()) {
3338 Elts[
Idx] = std::nullopt;
3342 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3347 return std::nullopt;
3348 Elts[
Idx] = *ExactInteger;
3361 unsigned IdxDiff =
Idx - PrevElt->second;
3362 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3370 int64_t Remainder = ValDiff % IdxDiff;
3372 if (Remainder != ValDiff) {
3375 return std::nullopt;
3381 SeqStepNum = ValDiff;
3382 else if (ValDiff != SeqStepNum)
3383 return std::nullopt;
3386 SeqStepDenom = IdxDiff;
3387 else if (IdxDiff != *SeqStepDenom)
3388 return std::nullopt;
3392 if (!PrevElt || PrevElt->first != *Elt)
3393 PrevElt = std::make_pair(*Elt,
Idx);
3397 if (!SeqStepNum || !SeqStepDenom)
3398 return std::nullopt;
3406 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3407 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3410 else if (Addend != SeqAddend)
3411 return std::nullopt;
3414 assert(SeqAddend &&
"Must have an addend if we have a step");
3416 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3437 MVT ContainerVT = VT;
3465 MVT VT =
Op.getSimpleValueType();
3477 unsigned MostCommonCount = 0;
3479 unsigned NumUndefElts =
3487 unsigned NumScalarLoads = 0;
3493 ValueCounts.
insert(std::make_pair(V, 0));
3494 unsigned &Count = ValueCounts[V];
3496 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3497 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3502 if (++Count >= MostCommonCount) {
3504 MostCommonCount = Count;
3508 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3509 unsigned NumDefElts = NumElts - NumUndefElts;
3510 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3516 ((MostCommonCount > DominantValueCountThreshold) ||
3529 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3530 LastOp != DominantValue) {
3539 Processed.insert(LastOp);
3544 const SDValue &V = OpIdx.value();
3545 if (V.isUndef() || !Processed.insert(V).second)
3547 if (ValueCounts[V] == 1) {
3556 return DAG.getConstant(V == V1, DL, XLenVT);
3572 MVT VT =
Op.getSimpleValueType();
3602 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3603 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3611 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3612 MVT IntegerViaVecVT =
3617 unsigned BitPos = 0, IntegerEltIdx = 0;
3620 for (
unsigned I = 0;
I < NumElts;) {
3622 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3623 Bits |= ((
uint64_t)BitValue << BitPos);
3629 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3630 if (NumViaIntegerBits <= 32)
3631 Bits = SignExtend64<32>(Bits);
3633 Elts[IntegerEltIdx] = Elt;
3642 if (NumElts < NumViaIntegerBits) {
3646 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3674 int64_t StepNumerator = SimpleVID->StepNumerator;
3675 unsigned StepDenominator = SimpleVID->StepDenominator;
3676 int64_t Addend = SimpleVID->Addend;
3678 assert(StepNumerator != 0 &&
"Invalid step");
3679 bool Negate =
false;
3680 int64_t SplatStepVal = StepNumerator;
3684 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3686 Negate = StepNumerator < 0;
3688 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3695 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3696 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3698 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3701 MVT VIDContainerVT =
3709 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3710 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3712 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3714 if (StepDenominator != 1) {
3719 if (Addend != 0 || Negate) {
3738 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3739 "Unexpected sequence type");
3743 unsigned ViaVecLen =
3747 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3750 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3751 const auto &SeqV = OpIdx.value();
3752 if (!SeqV.isUndef())
3754 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3759 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3760 SplatValue = SignExtend64<32>(SplatValue);
3782 const auto *BV = cast<BuildVectorSDNode>(
Op);
3785 BV->getRepeatedSequence(Sequence) &&
3786 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3787 unsigned SeqLen = Sequence.size();
3789 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3790 ViaIntVT == MVT::i64) &&
3791 "Unexpected sequence type");
3796 const unsigned RequiredVL = NumElts / SeqLen;
3797 const unsigned ViaVecLen =
3799 NumElts : RequiredVL;
3802 unsigned EltIdx = 0;
3803 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3807 for (
const auto &SeqV : Sequence) {
3808 if (!SeqV.isUndef())
3810 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3816 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3817 SplatValue = SignExtend64<32>(SplatValue);
3823 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3824 "Unexpected bitcast sequence");
3825 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3828 MVT ViaContainerVT =
3835 if (ViaVecLen != RequiredVL)
3852 if (EltBitSize - SignBits < 8) {
3856 Source, DAG, Subtarget);
3873 MVT VT =
Op.getSimpleValueType();
3945 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3949 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3965 unsigned NumUndefElts =
3967 unsigned NumDefElts = NumElts - NumUndefElts;
3968 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
3975 for (
unsigned i = 0; i < NumElts; i++) {
3977 if (i < NumElts / 2) {
3984 bool SelectMaskVal = (i < NumElts / 2);
3987 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
3988 MaskVals.
size() == NumElts);
4023 unsigned UndefCount = 0;
4030 LinearBudget -= PerSlideCost;
4033 LinearBudget -= PerSlideCost;
4036 LinearBudget -= PerSlideCost;
4039 if (LinearBudget < 0)
4044 "Illegal type which will result in reserved encoding");
4069 Vec,
Offset, Mask, VL, Policy);
4082 Vec,
Offset, Mask, VL, Policy);
4092 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4093 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4094 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4097 if ((LoC >> 31) == HiC)
4108 (isa<RegisterSDNode>(VL) &&
4109 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4111 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4126 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4127 Hi.getConstantOperandVal(1) == 31)
4146 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4158 bool HasPassthru = Passthru && !Passthru.
isUndef();
4159 if (!HasPassthru && !Passthru)
4167 if (Scalar.getValueType().bitsLE(XLenVT)) {
4174 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4178 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4179 "Unexpected scalar for splat lowering!");
4203 SDValue ExtractedVal = Scalar.getOperand(0);
4208 MVT ExtractedContainerVT = ExtractedVT;
4211 DAG, ExtractedContainerVT, Subtarget);
4213 ExtractedVal, DAG, Subtarget);
4215 if (ExtractedContainerVT.
bitsLE(VT))
4230 if (!Scalar.getValueType().bitsLE(XLenVT))
4233 VT,
DL, DAG, Subtarget);
4241 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4267 if (Src != V2.getOperand(0))
4271 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4276 V2.getConstantOperandVal(1) != Mask.size())
4280 if (Mask[0] != 0 && Mask[0] != 1)
4285 for (
unsigned i = 1; i != Mask.size(); ++i)
4286 if (Mask[i] != Mask[i - 1] + 2)
4304 int Size = Mask.size();
4306 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4312 EvenSrc = StartIndexes[0];
4313 OddSrc = StartIndexes[1];
4316 if (EvenSrc != 0 && OddSrc != 0)
4326 int HalfNumElts = NumElts / 2;
4327 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4343 int Size = Mask.size();
4355 for (
int i = 0; i !=
Size; ++i) {
4361 int StartIdx = i - (M %
Size);
4369 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4372 Rotation = CandidateRotation;
4373 else if (Rotation != CandidateRotation)
4378 int MaskSrc = M <
Size ? 0 : 1;
4383 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4388 TargetSrc = MaskSrc;
4389 else if (TargetSrc != MaskSrc)
4396 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4397 assert((LoSrc >= 0 || HiSrc >= 0) &&
4398 "Failed to find a rotated input vector!");
4413 MVT ContainerVT = VT;
4416 assert(Src.getSimpleValueType().isFixedLengthVector());
4420 MVT SrcContainerVT =
4433 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4440 unsigned Shift = EvenElts ? 0 : EltBits;
4446 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4472 auto findNonEXTRACT_SUBVECTORParent =
4473 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4478 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4479 Offset += Parent.getConstantOperandVal(1);
4480 Parent = Parent.getOperand(0);
4482 return std::make_pair(Parent,
Offset);
4485 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4486 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4495 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4496 if (NewMask[i] == -1)
4499 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4500 NewMask[i] = NewMask[i] + V1IndexOffset;
4504 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4510 if (NewMask[0] <= 0)
4514 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4515 if (NewMask[i - 1] + 1 != NewMask[i])
4519 MVT SrcVT = Src.getSimpleValueType();
4550 int NumSubElts,
Index;
4555 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4556 SDValue InPlace = OpsSwapped ? V2 : V1;
4557 SDValue ToInsert = OpsSwapped ? V1 : V2;
4567 if (NumSubElts +
Index >= (
int)NumElts)
4581 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4593 bool OpsSwapped =
false;
4594 if (!isa<BuildVectorSDNode>(V1)) {
4595 if (!isa<BuildVectorSDNode>(V2))
4600 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4608 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4609 for (
unsigned i = S; i != E; ++i)
4610 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4616 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4617 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4620 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4622 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4627 auto OpCode = IsVSlidedown ?
4632 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4635 Splat, TrueMask, VL);
4646 MVT VecContainerVT = VecVT;
4663 MVT WideContainerVT = WideVT;
4669 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4682 }
else if (Subtarget.hasStdExtZvbb()) {
4687 OffsetVec, Passthru, Mask, VL);
4690 Interleaved, EvenV, Passthru, Mask, VL);
4698 Interleaved, OffsetVec, Passthru, Mask, VL);
4706 OddV, Passthru, Mask, VL);
4712 OddV, AllOnesVec, Passthru, Mask, VL);
4720 Interleaved, OddsMul, Passthru, Mask, VL);
4727 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4773 if (ViaEltSize > NumElts)
4782 if (ViaEltSize > NumElts)
4788 if (ViaEltSize > NumElts)
4797 MVT &RotateVT,
unsigned &RotateAmt) {
4803 unsigned NumSubElts;
4805 NumElts, NumSubElts, RotateAmt))
4808 NumElts / NumSubElts);
4870 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4873 OutMasks(VRegsPerSrc, {-1, {}});
4878 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4879 int DstVecIdx = DstIdx / ElemsPerVReg;
4880 int DstSubIdx = DstIdx % ElemsPerVReg;
4881 int SrcIdx = Mask[DstIdx];
4882 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4884 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4885 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4886 if (OutMasks[DstVecIdx].first == -1)
4887 OutMasks[DstVecIdx].first = SrcVecIdx;
4888 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4894 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4895 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4909 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4910 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4911 if (SrcVecIdx == -1)
4913 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4920 unsigned InsertIdx = DstVecIdx * NumOpElts;
4933 MVT VT =
Op.getSimpleValueType();
4948 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4972 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4973 V = V.getOperand(
Offset / OpElements);
4979 auto *Ld = cast<LoadSDNode>(V);
4989 SDValue Ops[] = {Ld->getChain(),
5007 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5008 Ld->getPointerInfo().getWithOffset(
Offset),
5009 Ld->getOriginalAlign(),
5013 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5014 Ld->getOriginalAlign(),
5015 Ld->getMemOperand()->getFlags());
5026 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5029 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5051 if (Subtarget.hasStdExtZvkb())
5062 LoV = LoSrc == 0 ? V1 : V2;
5066 HiV = HiSrc == 0 ? V1 : V2;
5072 unsigned InvRotate = NumElts - Rotation;
5082 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5102 int EvenSrc, OddSrc;
5107 int Size = Mask.size();
5109 assert(EvenSrc >= 0 &&
"Undef source?");
5110 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5114 assert(OddSrc >= 0 &&
"Undef source?");
5115 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5124 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5133 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5162 MVT IndexContainerVT =
5167 for (
int MaskIndex : Mask) {
5168 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5177 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5191 int MaskIndex = MaskIdx.value();
5192 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5197 for (
int MaskIndex : Mask) {
5198 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5205 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5218 for (
int MaskIndex : Mask) {
5219 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5221 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5222 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5224 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5229 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5232 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5268RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5270 MVT VT =
Op.getSimpleValueType();
5274 MVT ContainerVT = VT;
5277 if (
Op->isVPOpcode()) {
5278 Mask =
Op.getOperand(1);
5282 VL =
Op.getOperand(2);
5288 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5290 FloatEltVT = MVT::f32;
5297 "Expected legal float type!");
5304 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5307 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5312 if (FloatVT.
bitsGT(VT)) {
5313 if (
Op->isVPOpcode())
5314 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5323 if (!
Op->isVPOpcode())
5327 MVT ContainerFloatVT =
5330 Src, Mask, RTZRM, VL);
5337 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5341 if (
Op->isVPOpcode()) {
5350 else if (IntVT.
bitsGT(VT))
5355 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5360 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5361 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5366 unsigned Adjust = ExponentBias + (EltSize - 1);
5368 if (
Op->isVPOpcode())
5378 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5379 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5389 MVT SrcVT =
Source.getSimpleValueType();
5398 SrcVT = ContainerVT;
5411 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5428 auto *
Load = cast<LoadSDNode>(
Op);
5429 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5432 Load->getMemoryVT(),
5433 *
Load->getMemOperand()))
5437 MVT VT =
Op.getSimpleValueType();
5439 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5440 "Unexpected unaligned RVV load type");
5444 "Expecting equally-sized RVV vector types to be legal");
5446 Load->getPointerInfo(),
Load->getOriginalAlign(),
5447 Load->getMemOperand()->getFlags());
5457 auto *
Store = cast<StoreSDNode>(
Op);
5458 assert(Store &&
Store->getValue().getValueType().isVector() &&
5459 "Expected vector store");
5462 Store->getMemoryVT(),
5463 *
Store->getMemOperand()))
5470 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5471 "Unexpected unaligned RVV store type");
5475 "Expecting equally-sized RVV vector types to be legal");
5476 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5478 Store->getPointerInfo(),
Store->getOriginalAlign(),
5479 Store->getMemOperand()->getFlags());
5484 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5486 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5513 unsigned ShiftAmt, AddOpc;
5530 if (Subtarget.hasStdExtZtso()) {
5553 "Unexpected custom legalisation");
5574 "Unexpected custom legalisation");
5589 "Unexpected custom legalisation");
5590 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5610 "Unexpected custom legalisation");
5626 MVT VT =
Op.getSimpleValueType();
5628 unsigned Check =
Op.getConstantOperandVal(1);
5629 unsigned TDCMask = 0;
5657 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5662 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5664 VL =
Op.getOperand(3);
5667 VL,
Op->getFlags());
5682 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5684 MVT MaskContainerVT =
5687 VL =
Op.getOperand(3);
5692 Mask, VL,
Op->getFlags());
5695 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5700 DAG.
getUNDEF(ContainerVT), Mask, VL});
5704 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5708 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5712 DAG.
getUNDEF(ContainerVT), Mask, VL});
5728 MVT VT =
Op.getSimpleValueType();
5755 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5762 MVT ContainerVT = VT;
5770 if (
Op->isVPOpcode()) {
5771 Mask =
Op.getOperand(2);
5775 VL =
Op.getOperand(3);
5783 {X, X, DAG.getCondCode(ISD::SETOEQ),
5784 DAG.getUNDEF(ContainerVT), Mask, VL});
5792 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5793 DAG.getUNDEF(ContainerVT), Mask, VL});
5803 DAG.
getUNDEF(ContainerVT), Mask, VL);
5811#define OP_CASE(NODE) \
5813 return RISCVISD::NODE##_VL;
5814#define VP_CASE(NODE) \
5815 case ISD::VP_##NODE: \
5816 return RISCVISD::NODE##_VL;
5818 switch (
Op.getOpcode()) {
5894 case ISD::VP_CTLZ_ZERO_UNDEF:
5897 case ISD::VP_CTTZ_ZERO_UNDEF:
5906 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5911 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5916 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5919 case ISD::VP_SELECT:
5928 case ISD::VP_SIGN_EXTEND:
5930 case ISD::VP_ZERO_EXTEND:
5932 case ISD::VP_FP_TO_SINT:
5934 case ISD::VP_FP_TO_UINT:
5937 case ISD::VP_FMINNUM:
5940 case ISD::VP_FMAXNUM:
5945 case ISD::VP_LLRINT:
5957 "not a RISC-V target specific op");
5963 "adding target specific op should update this function");
5983 "not a RISC-V target specific op");
5989 "adding target specific op should update this function");
6008 if (!
Op.getOperand(j).getValueType().isVector()) {
6009 LoOperands[j] =
Op.getOperand(j);
6010 HiOperands[j] =
Op.getOperand(j);
6013 std::tie(LoOperands[j], HiOperands[j]) =
6018 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6020 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6035 std::tie(LoOperands[j], HiOperands[j]) =
6039 if (!
Op.getOperand(j).getValueType().isVector()) {
6040 LoOperands[j] =
Op.getOperand(j);
6041 HiOperands[j] =
Op.getOperand(j);
6044 std::tie(LoOperands[j], HiOperands[j]) =
6049 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6051 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6061 auto [EVLLo, EVLHi] =
6062 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6066 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6068 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6086 if (!
Op.getOperand(j).getValueType().isVector()) {
6087 LoOperands[j] =
Op.getOperand(j);
6088 HiOperands[j] =
Op.getOperand(j);
6091 std::tie(LoOperands[j], HiOperands[j]) =
6096 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6099 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6108 switch (
Op.getOpcode()) {
6114 return lowerGlobalAddress(
Op, DAG);
6116 return lowerBlockAddress(
Op, DAG);
6118 return lowerConstantPool(
Op, DAG);
6120 return lowerJumpTable(
Op, DAG);
6122 return lowerGlobalTLSAddress(
Op, DAG);
6126 return lowerSELECT(
Op, DAG);
6128 return lowerBRCOND(
Op, DAG);
6130 return lowerVASTART(
Op, DAG);
6132 return lowerFRAMEADDR(
Op, DAG);
6134 return lowerRETURNADDR(
Op, DAG);
6141 return lowerShiftLeftParts(
Op, DAG);
6143 return lowerShiftRightParts(
Op, DAG,
true);
6145 return lowerShiftRightParts(
Op, DAG,
false);
6148 if (
Op.getValueType().isFixedLengthVector()) {
6149 assert(Subtarget.hasStdExtZvkb());
6150 return lowerToScalableOp(
Op, DAG);
6152 assert(Subtarget.hasVendorXTHeadBb() &&
6153 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6154 "Unexpected custom legalization");
6156 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6161 EVT VT =
Op.getValueType();
6165 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6171 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6172 Subtarget.hasStdExtZfbfmin()) {
6177 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6184 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6201 "Unexpected types");
6235 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6237 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6239 return LowerINTRINSIC_VOID(
Op, DAG);
6241 return LowerIS_FPCLASS(
Op, DAG);
6243 MVT VT =
Op.getSimpleValueType();
6245 assert(Subtarget.hasStdExtZvbb());
6246 return lowerToScalableOp(
Op, DAG);
6249 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6257 if (!
Op.getSimpleValueType().isVector())
6259 return lowerVectorTruncLike(
Op, DAG);
6262 if (
Op.getOperand(0).getValueType().isVector() &&
6263 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6264 return lowerVectorMaskExt(
Op, DAG, 1);
6267 if (
Op.getOperand(0).getValueType().isVector() &&
6268 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6269 return lowerVectorMaskExt(
Op, DAG, -1);
6272 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6274 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6276 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6278 MVT VT =
Op.getSimpleValueType();
6286 MVT ContainerVT = VT;
6292 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6299 MVT VT =
Op.getSimpleValueType();
6319 }
else if ((Val % 8) == 0) {
6335 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6336 Op.getOperand(1).getValueType() == MVT::i32) {
6348 if (
Op.getValueType() == MVT::nxv32f16 &&
6355 EVT VT =
Op.getValueType();
6358 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6360 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6366 if (!
Op.getValueType().isVector())
6368 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6372 EVT VT =
Op.getValueType();
6375 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6377 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6385 if (!
Op.getValueType().isVector())
6387 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6391 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6394 if (
Op.getValueType().isVector() &&
6395 Op.getValueType().getScalarType() == MVT::f16 &&
6398 if (
Op.getValueType() == MVT::nxv32f16)
6413 Op1.getValueType().isVector() &&
6414 Op1.getValueType().getScalarType() == MVT::f16 &&
6417 if (Op1.getValueType() == MVT::nxv32f16)
6422 Op1.getValueType().getVectorElementCount());
6425 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6435 MVT VT =
Op.getSimpleValueType();
6439 bool IsStrict =
Op->isStrictFPOpcode();
6440 SDValue Src =
Op.getOperand(0 + IsStrict);
6442 MVT SrcVT = Src.getSimpleValueType();
6447 "Unexpected vector element types");
6451 if (EltSize > (2 * SrcEltSize)) {
6463 Op.getOperand(0), Ext);
6467 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6472 auto [FExt, Chain] =
6474 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6481 if (SrcEltSize > (2 * EltSize)) {
6484 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6489 Op.getOperand(0), Src);
6504 Op.getOperand(0), Src);
6518 unsigned RVVOpc = 0;
6519 switch (
Op.getOpcode()) {
6551 "Expected same element count");
6558 Op.getOperand(0), Src, Mask, VL);
6562 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6577 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6584 MVT VT =
Op.getSimpleValueType();
6606 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6622 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6642 return lowerVECREDUCE(
Op, DAG);
6646 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6647 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6648 return lowerVECREDUCE(
Op, DAG);
6655 return lowerFPVECREDUCE(
Op, DAG);
6656 case ISD::VP_REDUCE_ADD:
6657 case ISD::VP_REDUCE_UMAX:
6658 case ISD::VP_REDUCE_SMAX:
6659 case ISD::VP_REDUCE_UMIN:
6660 case ISD::VP_REDUCE_SMIN:
6661 case ISD::VP_REDUCE_FADD:
6662 case ISD::VP_REDUCE_SEQ_FADD:
6663 case ISD::VP_REDUCE_FMIN:
6664 case ISD::VP_REDUCE_FMAX:
6665 case ISD::VP_REDUCE_FMINIMUM:
6666 case ISD::VP_REDUCE_FMAXIMUM:
6667 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6671 return lowerVPREDUCE(
Op, DAG);
6672 case ISD::VP_REDUCE_AND:
6673 case ISD::VP_REDUCE_OR:
6674 case ISD::VP_REDUCE_XOR:
6675 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6676 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6677 return lowerVPREDUCE(
Op, DAG);
6678 case ISD::VP_CTTZ_ELTS:
6679 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6680 return lowerVPCttzElements(
Op, DAG);
6684 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6687 return lowerINSERT_SUBVECTOR(
Op, DAG);
6689 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6691 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6693 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6695 return lowerSTEP_VECTOR(
Op, DAG);
6697 return lowerVECTOR_REVERSE(
Op, DAG);
6699 return lowerVECTOR_SPLICE(
Op, DAG);
6703 if (
Op.getValueType().getScalarType() == MVT::f16 &&
6706 if (
Op.getValueType() == MVT::nxv32f16)
6718 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6719 return lowerVectorMaskSplat(
Op, DAG);
6727 MVT VT =
Op.getSimpleValueType();
6728 MVT ContainerVT = VT;
6746 Op->ops().take_front(HalfNumOps));
6748 Op->ops().drop_front(HalfNumOps));
6752 unsigned NumOpElts =
6753 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6756 SDValue SubVec = OpIdx.value();
6767 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6769 if (
Op.getValueType().isFixedLengthVector())
6770 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6773 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6775 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6776 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6780 return lowerMaskedLoad(
Op, DAG);
6783 return lowerMaskedStore(
Op, DAG);
6792 EVT VT =
Op.getValueType();
6803 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6805 MVT VT =
Op.getSimpleValueType();
6810 "Unexpected CondCode");
6818 if (isa<ConstantSDNode>(
RHS)) {
6819 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6820 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6839 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6844 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6860 return lowerToScalableOp(
Op, DAG);
6864 if (
Op.getSimpleValueType().isFixedLengthVector())
6865 return lowerToScalableOp(
Op, DAG);
6867 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6868 "Unexpected custom legalisation");
6880 if (
Op.getValueType() == MVT::nxv32f16 &&
6891 return lowerToScalableOp(
Op, DAG);
6894 if (!
Op.getValueType().isVector())
6896 return lowerToScalableOp(
Op, DAG);
6899 if (!
Op.getValueType().isVector())
6901 return lowerToScalableOp(
Op, DAG);
6905 EVT VT =
Op->getValueType(0);
6920 return lowerABS(
Op, DAG);
6925 if (Subtarget.hasStdExtZvbb())
6926 return lowerToScalableOp(
Op, DAG);
6928 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6930 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6932 if (
Op.getValueType() == MVT::nxv32f16 &&
6936 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6943 if (
Op.getValueType() == MVT::nxv32f16 &&
6947 return lowerToScalableOp(
Op, DAG);
6950 return lowerVectorStrictFSetcc(
Op, DAG);
6960 case ISD::VP_GATHER:
6961 return lowerMaskedGather(
Op, DAG);
6963 case ISD::VP_SCATTER:
6964 return lowerMaskedScatter(
Op, DAG);
6966 return lowerGET_ROUNDING(
Op, DAG);
6968 return lowerSET_ROUNDING(
Op, DAG);
6970 return lowerEH_DWARF_CFA(
Op, DAG);
6971 case ISD::VP_SELECT:
6980 case ISD::VP_UADDSAT:
6981 case ISD::VP_USUBSAT:
6982 case ISD::VP_SADDSAT:
6983 case ISD::VP_SSUBSAT:
6985 case ISD::VP_LLRINT:
6986 return lowerVPOp(
Op, DAG);
6990 return lowerLogicVPOp(
Op, DAG);
6999 case ISD::VP_FMINNUM:
7000 case ISD::VP_FMAXNUM:
7001 case ISD::VP_FCOPYSIGN:
7002 if (
Op.getValueType() == MVT::nxv32f16 &&
7010 return lowerVPOp(
Op, DAG);
7011 case ISD::VP_IS_FPCLASS:
7012 return LowerIS_FPCLASS(
Op, DAG);
7013 case ISD::VP_SIGN_EXTEND:
7014 case ISD::VP_ZERO_EXTEND:
7015 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7016 return lowerVPExtMaskOp(
Op, DAG);
7017 return lowerVPOp(
Op, DAG);
7018 case ISD::VP_TRUNCATE:
7019 return lowerVectorTruncLike(
Op, DAG);
7020 case ISD::VP_FP_EXTEND:
7021 case ISD::VP_FP_ROUND:
7022 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7023 case ISD::VP_SINT_TO_FP:
7024 case ISD::VP_UINT_TO_FP:
7025 if (
Op.getValueType().isVector() &&
7026 Op.getValueType().getScalarType() == MVT::f16 &&
7029 if (
Op.getValueType() == MVT::nxv32f16)
7041 case ISD::VP_FP_TO_SINT:
7042 case ISD::VP_FP_TO_UINT:
7044 Op1.getValueType().isVector() &&
7045 Op1.getValueType().getScalarType() == MVT::f16 &&
7048 if (Op1.getValueType() == MVT::nxv32f16)
7053 Op1.getValueType().getVectorElementCount());
7057 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7059 return lowerVPFPIntConvOp(
Op, DAG);
7061 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7065 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7066 return lowerVPSetCCMaskOp(
Op, DAG);
7072 case ISD::VP_BITREVERSE:
7074 return lowerVPOp(
Op, DAG);
7076 case ISD::VP_CTLZ_ZERO_UNDEF:
7077 if (Subtarget.hasStdExtZvbb())
7078 return lowerVPOp(
Op, DAG);
7079 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7081 case ISD::VP_CTTZ_ZERO_UNDEF:
7082 if (Subtarget.hasStdExtZvbb())
7083 return lowerVPOp(
Op, DAG);
7084 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7086 return lowerVPOp(
Op, DAG);
7087 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7088 return lowerVPStridedLoad(
Op, DAG);
7089 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7090 return lowerVPStridedStore(
Op, DAG);
7092 case ISD::VP_FFLOOR:
7094 case ISD::VP_FNEARBYINT:
7095 case ISD::VP_FROUND:
7096 case ISD::VP_FROUNDEVEN:
7097 case ISD::VP_FROUNDTOZERO:
7098 if (
Op.getValueType() == MVT::nxv32f16 &&
7103 case ISD::VP_FMAXIMUM:
7104 case ISD::VP_FMINIMUM:
7105 if (
Op.getValueType() == MVT::nxv32f16 &&
7110 case ISD::EXPERIMENTAL_VP_SPLICE:
7111 return lowerVPSpliceExperimental(
Op, DAG);
7112 case ISD::EXPERIMENTAL_VP_REVERSE:
7113 return lowerVPReverseExperimental(
Op, DAG);
7131 N->getOffset(), Flags);
7139template <
class NodeTy>
7141 bool IsLocal,
bool IsExternWeak)
const {
7151 if (IsLocal && !Subtarget.allowTaggedGlobals())
7213 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7222 return getAddr(
N, DAG);
7229 return getAddr(
N, DAG);
7236 return getAddr(
N, DAG);
7241 bool UseGOT)
const {
7305 Args.push_back(Entry);
7338 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7352 Addr = getStaticTLSAddr(
N, DAG,
false);
7355 Addr = getStaticTLSAddr(
N, DAG,
true);
7360 : getDynamicTLSAddr(
N, DAG);
7377 if (
LHS == LHS2 &&
RHS == RHS2) {
7382 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7390 return std::nullopt;
7398 MVT VT =
N->getSimpleValueType(0);
7428 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7431 if (~TrueVal == FalseVal) {
7471 if (Subtarget.hasShortForwardBranchOpt())
7474 unsigned SelOpNo = 0;
7484 unsigned ConstSelOpNo = 1;
7485 unsigned OtherSelOpNo = 2;
7486 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7491 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7492 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7496 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7497 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7503 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7505 std::swap(NewConstOps[0], NewConstOps[1]);
7517 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7519 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7522 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7523 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7532 MVT VT =
Op.getSimpleValueType();
7546 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7574 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7578 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7580 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7581 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7583 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7588 DL, VT, LHSVal, CondV);
7604 if (
Op.hasOneUse()) {
7605 unsigned UseOpc =
Op->use_begin()->getOpcode();
7611 return lowerSELECT(NewSel, DAG);
7639 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7660 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7664 if (TrueVal - 1 == FalseVal)
7666 if (TrueVal + 1 == FalseVal)
7673 RHS == TrueV && LHS == FalseV) {
7690 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7716 LHS, RHS, TargetCC,
Op.getOperand(2));
7734 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7746 int XLenInBytes = Subtarget.
getXLen() / 8;
7748 EVT VT =
Op.getValueType();
7751 unsigned Depth =
Op.getConstantOperandVal(0);
7753 int Offset = -(XLenInBytes * 2);
7769 int XLenInBytes = Subtarget.
getXLen() / 8;
7774 EVT VT =
Op.getValueType();
7776 unsigned Depth =
Op.getConstantOperandVal(0);
7778 int Off = -XLenInBytes;
7779 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7798 EVT VT =
Lo.getValueType();
7837 EVT VT =
Lo.getValueType();
7888 MVT VT =
Op.getSimpleValueType();
7914 MVT VecVT =
Op.getSimpleValueType();
7916 "Unexpected SPLAT_VECTOR_PARTS lowering");
7922 MVT ContainerVT = VecVT;
7942 int64_t ExtTrueVal)
const {
7944 MVT VecVT =
Op.getSimpleValueType();
7947 assert(Src.getValueType().isVector() &&
7948 Src.getValueType().getVectorElementType() == MVT::i1);
7969 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7971 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7974 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7979SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7981 MVT ExtVT =
Op.getSimpleValueType();
7985 MVT VT =
Op.getOperand(0).getSimpleValueType();
8011 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8013 EVT MaskVT =
Op.getValueType();
8016 "Unexpected type for vector mask lowering");
8018 MVT VecVT = Src.getSimpleValueType();
8022 VL =
Op.getOperand(2);
8025 MVT ContainerVT = VecVT;
8031 MVT MaskContainerVT =
8038 std::tie(Mask, VL) =
8046 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8048 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8052 DAG.
getUNDEF(ContainerVT), Mask, VL);
8055 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8063 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8066 MVT VT =
Op.getSimpleValueType();
8068 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8072 return lowerVectorMaskTruncLike(
Op, DAG);
8080 MVT SrcVT = Src.getSimpleValueType();
8085 "Unexpected vector truncate lowering");
8087 MVT ContainerVT = SrcVT;
8091 VL =
Op.getOperand(2);
8104 std::tie(Mask, VL) =
8115 }
while (SrcEltVT != DstEltVT);
8124RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8129 MVT VT =
Op.getSimpleValueType();
8130 MVT SrcVT = Src.getSimpleValueType();
8131 MVT ContainerVT = VT;
8154 Chain, Src, Mask, VL);
8155 Chain = Src.getValue(1);
8162 Chain, Src, Mask, VL);
8173RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8176 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8183 MVT VT =
Op.getSimpleValueType();
8185 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8188 MVT SrcVT = Src.getSimpleValueType();
8190 bool IsDirectExtend =
8198 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8201 MVT ContainerVT = VT;
8205 VL =
Op.getOperand(2);
8219 std::tie(Mask, VL) =
8225 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8231 unsigned InterConvOpc =
8236 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8238 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8249static std::optional<MVT>
8255 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8257 if (MaxIdx < MinVLMAX)
8259 else if (MaxIdx < MinVLMAX * 2)
8261 else if (MaxIdx < MinVLMAX * 4)
8266 return std::nullopt;
8279 MVT VecVT =
Op.getSimpleValueType();
8293 MVT ContainerVT = VecVT;
8302 MVT OrigContainerVT = ContainerVT;
8305 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8306 const unsigned OrigIdx = IdxC->getZExtValue();
8309 DL, DAG, Subtarget)) {
8310 ContainerVT = *ShrunkVT;
8319 VLEN && ContainerVT.
bitsGT(M1VT)) {
8322 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8323 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8324 unsigned ExtractIdx =
8343 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8344 const auto *CVal = cast<ConstantSDNode>(Val);
8345 if (isInt<32>(CVal->getSExtValue())) {
8346 IsLegalInsert =
true;
8355 if (IsLegalInsert) {
8361 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8377 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8378 MVT I32ContainerVT =
8389 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8394 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8396 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8401 ValInVec, AlignedIdx);
8411 DAG.
getUNDEF(I32ContainerVT), ValLo,
8412 I32Mask, InsertI64VL);
8414 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8415 I32Mask, InsertI64VL);
8417 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8430 Idx, Mask, InsertVL, Policy);
8434 Slideup, AlignedIdx);
8449 EVT EltVT =
Op.getValueType();
8456 MVT ContainerVT = VecVT;
8472 unsigned WidenVecLen;
8475 unsigned MaxEEW = Subtarget.
getELen();
8480 "the number of elements should be power of 2");
8484 ExtractBitIdx =
Idx;
8486 WideEltVT = LargestEltVT;
8489 ExtractElementIdx = DAG.
getNode(
8500 Vec, ExtractElementIdx);
8516 MVT ContainerVT = VecVT;
8527 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8528 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8530 unsigned OrigIdx = IdxC->getZExtValue();
8533 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8534 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8535 unsigned ExtractIdx =
8545 std::optional<uint64_t> MaxIdx;
8548 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8549 MaxIdx = IdxC->getZExtValue();
8551 if (
auto SmallerVT =
8553 ContainerVT = *SmallerVT;
8600 "Unexpected opcode");
8607 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8612 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8629 if (OpVT.
bitsLT(XLenVT)) {
8636 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8647 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8650 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8661 case Intrinsic::riscv_vslide1up:
8662 case Intrinsic::riscv_vslide1down:
8663 case Intrinsic::riscv_vslide1up_mask:
8664 case Intrinsic::riscv_vslide1down_mask: {
8667 bool IsMasked = NumOps == 7;
8673 std::tie(ScalarLo, ScalarHi) =
8681 if (isa<ConstantSDNode>(AVL)) {
8682 const auto [MinVLMAX, MaxVLMAX] =
8686 if (AVLInt <= MinVLMAX) {
8688 }
else if (AVLInt >= 2 * MaxVLMAX) {
8695 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8729 if (IntNo == Intrinsic::riscv_vslide1up ||
8730 IntNo == Intrinsic::riscv_vslide1up_mask) {
8732 ScalarHi, I32Mask, I32VL);
8734 ScalarLo, I32Mask, I32VL);
8737 ScalarLo, I32Mask, I32VL);
8739 ScalarHi, I32Mask, I32VL);
8788 const unsigned ElementWidth = 8;
8793 [[maybe_unused]]
unsigned MinVF =
8796 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8800 bool Fractional = VF < LMul1VF;
8801 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8822 MVT ContainerVT = OpVT;
8849 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8853 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8870 if (OpVT.
bitsLT(XLenVT)) {
8873 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8886 EVT ValType = V.getValueType();
8887 if (ValType.isVector() && ValType.isFloatingPoint()) {
8890 ValType.getVectorElementCount());
8893 if (ValType.isFixedLengthVector()) {
8895 DAG, V.getSimpleValueType(), Subtarget);
8911 unsigned IntNo =
Op.getConstantOperandVal(0);
8918 case Intrinsic::thread_pointer: {
8922 case Intrinsic::riscv_orc_b:
8923 case Intrinsic::riscv_brev8:
8924 case Intrinsic::riscv_sha256sig0:
8925 case Intrinsic::riscv_sha256sig1:
8926 case Intrinsic::riscv_sha256sum0:
8927 case Intrinsic::riscv_sha256sum1:
8928 case Intrinsic::riscv_sm3p0:
8929 case Intrinsic::riscv_sm3p1: {
8949 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8951 case Intrinsic::riscv_sm4ks:
8952 case Intrinsic::riscv_sm4ed: {
8962 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
8966 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8969 case Intrinsic::riscv_zip:
8970 case Intrinsic::riscv_unzip: {
8973 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8975 case Intrinsic::riscv_mopr: {
8988 case Intrinsic::riscv_moprr: {
9000 Op.getOperand(2),
Op.getOperand(3));
9002 case Intrinsic::riscv_clmul:
9013 case Intrinsic::riscv_clmulh:
9014 case Intrinsic::riscv_clmulr: {
9032 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9034 case Intrinsic::experimental_get_vector_length:
9036 case Intrinsic::experimental_cttz_elts:
9038 case Intrinsic::riscv_vmv_x_s: {
9042 case Intrinsic::riscv_vfmv_f_s:
9045 case Intrinsic::riscv_vmv_v_x:
9047 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9049 case Intrinsic::riscv_vfmv_v_f:
9051 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9052 case Intrinsic::riscv_vmv_s_x: {
9055 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9058 Op.getOperand(1), Scalar,
Op.getOperand(3));
9061 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9078 MVT VT =
Op.getSimpleValueType();
9083 if (
Op.getOperand(1).isUndef())
9099 case Intrinsic::riscv_vfmv_s_f:
9101 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9103 case Intrinsic::riscv_vaesdf_vv:
9104 case Intrinsic::riscv_vaesdf_vs:
9105 case Intrinsic::riscv_vaesdm_vv:
9106 case Intrinsic::riscv_vaesdm_vs:
9107 case Intrinsic::riscv_vaesef_vv:
9108 case Intrinsic::riscv_vaesef_vs:
9109 case Intrinsic::riscv_vaesem_vv:
9110 case Intrinsic::riscv_vaesem_vs:
9111 case Intrinsic::riscv_vaeskf1:
9112 case Intrinsic::riscv_vaeskf2:
9113 case Intrinsic::riscv_vaesz_vs:
9114 case Intrinsic::riscv_vsm4k:
9115 case Intrinsic::riscv_vsm4r_vv:
9116 case Intrinsic::riscv_vsm4r_vs: {
9117 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9118 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9119 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9124 case Intrinsic::riscv_vsm3c:
9125 case Intrinsic::riscv_vsm3me: {
9126 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9127 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9132 case Intrinsic::riscv_vsha2ch:
9133 case Intrinsic::riscv_vsha2cl:
9134 case Intrinsic::riscv_vsha2ms: {
9135 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9136 !Subtarget.hasStdExtZvknhb())
9138 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9139 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9140 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9144 case Intrinsic::riscv_sf_vc_v_x:
9145 case Intrinsic::riscv_sf_vc_v_i:
9146 case Intrinsic::riscv_sf_vc_v_xv:
9147 case Intrinsic::riscv_sf_vc_v_iv:
9148 case Intrinsic::riscv_sf_vc_v_vv:
9149 case Intrinsic::riscv_sf_vc_v_fv:
9150 case Intrinsic::riscv_sf_vc_v_xvv:
9151 case Intrinsic::riscv_sf_vc_v_ivv:
9152 case Intrinsic::riscv_sf_vc_v_vvv:
9153 case Intrinsic::riscv_sf_vc_v_fvv:
9154 case Intrinsic::riscv_sf_vc_v_xvw:
9155 case Intrinsic::riscv_sf_vc_v_ivw:
9156 case Intrinsic::riscv_sf_vc_v_vvw:
9157 case Intrinsic::riscv_sf_vc_v_fvw: {
9158 MVT VT =
Op.getSimpleValueType();
9195 MVT VT =
Op.getSimpleValueType();
9199 if (VT.isFloatingPoint()) {
9204 if (VT.isFixedLengthVector())
9214 if (VT.isFixedLengthVector())
9216 if (VT.isFloatingPoint())
9235 unsigned IntNo =
Op.getConstantOperandVal(1);
9239 case Intrinsic::riscv_masked_strided_load: {
9248 MVT VT =
Op->getSimpleValueType(0);
9249 MVT ContainerVT = VT;
9262 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9274 ScalarVT,
Load->getMemOperand());
9280 Load->getMemOperand());
9285 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9290 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9292 Ops.push_back(PassThru);
9294 Ops.push_back(Stride);
9296 Ops.push_back(Mask);
9301 Ops.push_back(Policy);
9307 Load->getMemoryVT(),
Load->getMemOperand());
9308 Chain =
Result.getValue(1);
9314 case Intrinsic::riscv_seg2_load:
9315 case Intrinsic::riscv_seg3_load:
9316 case Intrinsic::riscv_seg4_load:
9317 case Intrinsic::riscv_seg5_load:
9318 case Intrinsic::riscv_seg6_load:
9319 case Intrinsic::riscv_seg7_load:
9320 case Intrinsic::riscv_seg8_load: {
9323 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9324 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9325 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9326 Intrinsic::riscv_vlseg8};
9327 unsigned NF =
Op->getNumValues() - 1;
9328 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9330 MVT VT =
Op->getSimpleValueType(0);
9336 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9338 ContainerVTs.push_back(MVT::Other);
9346 Load->getMemoryVT(),
Load->getMemOperand());
9348 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9354 case Intrinsic::riscv_sf_vc_v_x_se:
9356 case Intrinsic::riscv_sf_vc_v_i_se:
9358 case Intrinsic::riscv_sf_vc_v_xv_se:
9360 case Intrinsic::riscv_sf_vc_v_iv_se:
9362 case Intrinsic::riscv_sf_vc_v_vv_se:
9364 case Intrinsic::riscv_sf_vc_v_fv_se:
9366 case Intrinsic::riscv_sf_vc_v_xvv_se:
9368 case Intrinsic::riscv_sf_vc_v_ivv_se:
9370 case Intrinsic::riscv_sf_vc_v_vvv_se:
9372 case Intrinsic::riscv_sf_vc_v_fvv_se:
9374 case Intrinsic::riscv_sf_vc_v_xvw_se:
9376 case Intrinsic::riscv_sf_vc_v_ivw_se:
9378 case Intrinsic::riscv_sf_vc_v_vvw_se:
9380 case Intrinsic::riscv_sf_vc_v_fvw_se:
9389 unsigned IntNo =
Op.getConstantOperandVal(1);
9393 case Intrinsic::riscv_masked_strided_store: {
9404 MVT ContainerVT = VT;
9418 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9421 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9431 Ops,
Store->getMemoryVT(),
9432 Store->getMemOperand());
9434 case Intrinsic::riscv_seg2_store:
9435 case Intrinsic::riscv_seg3_store:
9436 case Intrinsic::riscv_seg4_store:
9437 case Intrinsic::riscv_seg5_store:
9438 case Intrinsic::riscv_seg6_store:
9439 case Intrinsic::riscv_seg7_store:
9440 case Intrinsic::riscv_seg8_store: {
9443 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9444 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9445 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9446 Intrinsic::riscv_vsseg8};
9449 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9451 MVT VT =
Op->getOperand(2).getSimpleValueType();
9459 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9461 for (
unsigned i = 0; i < NF; i++)
9463 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9468 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9470 case Intrinsic::riscv_sf_vc_xv_se:
9472 case Intrinsic::riscv_sf_vc_iv_se:
9474 case Intrinsic::riscv_sf_vc_vv_se:
9476 case Intrinsic::riscv_sf_vc_fv_se:
9478 case Intrinsic::riscv_sf_vc_xvv_se:
9480 case Intrinsic::riscv_sf_vc_ivv_se:
9482 case Intrinsic::riscv_sf_vc_vvv_se:
9484 case Intrinsic::riscv_sf_vc_fvv_se:
9486 case Intrinsic::riscv_sf_vc_xvw_se:
9488 case Intrinsic::riscv_sf_vc_ivw_se:
9490 case Intrinsic::riscv_sf_vc_vvw_se:
9492 case Intrinsic::riscv_sf_vc_fvw_se:
9500 switch (ISDOpcode) {
9503 case ISD::VP_REDUCE_ADD:
9506 case ISD::VP_REDUCE_UMAX:
9509 case ISD::VP_REDUCE_SMAX:
9512 case ISD::VP_REDUCE_UMIN:
9515 case ISD::VP_REDUCE_SMIN:
9518 case ISD::VP_REDUCE_AND:
9521 case ISD::VP_REDUCE_OR:
9524 case ISD::VP_REDUCE_XOR:
9527 case ISD::VP_REDUCE_FADD:
9529 case ISD::VP_REDUCE_SEQ_FADD:
9531 case ISD::VP_REDUCE_FMAX:
9532 case ISD::VP_REDUCE_FMAXIMUM:
9534 case ISD::VP_REDUCE_FMIN:
9535 case ISD::VP_REDUCE_FMINIMUM:
9545 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9550 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9551 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9552 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9553 "Unexpected reduction lowering");
9557 MVT ContainerVT = VecVT;
9566 VL =
Op.getOperand(3);
9568 std::tie(Mask, VL) =
9576 switch (
Op.getOpcode()) {
9580 case ISD::VP_REDUCE_AND: {
9590 case ISD::VP_REDUCE_OR:
9597 case ISD::VP_REDUCE_XOR: {
9620 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9624 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9625 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9626 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9627 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9643 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9647 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9650 if (M1VT != InnerVT)
9656 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9675 VecEVT =
Lo.getValueType();
9688 MVT ContainerVT = VecVT;
9708 Mask, VL,
DL, DAG, Subtarget);
9714static std::tuple<unsigned, SDValue, SDValue>
9718 auto Flags =
Op->getFlags();
9719 unsigned Opcode =
Op.getOpcode();
9743 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9751 MVT VecEltVT =
Op.getSimpleValueType();
9755 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9759 MVT ContainerVT = VecVT;
9765 MVT ResVT =
Op.getSimpleValueType();
9768 VL,
DL, DAG, Subtarget);
9773 if (
Op->getFlags().hasNoNaNs())
9779 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9780 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9786 DL, ResVT, NoNaNs, Res,
9794 unsigned Opc =
Op.getOpcode();
9817 Vec, Mask, VL,
DL, DAG, Subtarget);
9818 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9819 Op->getFlags().hasNoNaNs())
9836 DL, ResVT, NoNaNs, Res,
9850 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9859 (OrigIdx != 0 || !Vec.
isUndef())) {
9862 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9865 "Unexpected mask vector lowering");
9899 MVT ContainerVT = VecVT;
9935 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9936 SlideupAmt, Mask, VL, Policy);
9944 MVT ContainerVecVT = VecVT;
9950 MVT ContainerSubVecVT = SubVecVT;
9966 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
9967 SubRegIdx = Decompose.first;
9969 (OrigIdx % Vscale));
9973 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
9974 SubRegIdx = Decompose.first;
9981 bool ExactlyVecRegSized =
9983 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
9998 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10002 if (SubRegIdx == RISCV::NoSubRegister) {
10021 MVT InterSubVT = ContainerVecVT;
10022 SDValue AlignedExtract = Vec;
10060 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10061 SlideupAmt, Mask, VL, Policy);
10066 if (ContainerVecVT.
bitsGT(InterSubVT))
10075 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10081 MVT SubVecVT =
Op.getSimpleValueType();
10086 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10097 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10100 "Unexpected mask vector lowering");
10139 MVT ContainerVT = VecVT;
10147 if (
auto ShrunkVT =
10149 ContainerVT = *ShrunkVT;
10163 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10175 MVT ContainerSubVecVT = SubVecVT;
10179 unsigned SubRegIdx;
10189 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10190 SubRegIdx = Decompose.first;
10192 (OrigIdx % Vscale));
10196 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10197 SubRegIdx = Decompose.first;
10220 MVT InterSubVT = VecVT;
10224 assert(SubRegIdx != RISCV::NoSubRegister);
10238 Vec, SlidedownAmt, Mask, VL);
10247 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10254 MVT VT =
N.getSimpleValueType();
10258 assert(
Op.getSimpleValueType() == VT &&
10259 "Operands and result must be same type");
10263 unsigned NumVals =
N->getNumValues();
10266 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10269 for (
unsigned I = 0;
I < NumVals;
I++) {
10275 if (TruncVals.
size() > 1)
10277 return TruncVals.
front();
10283 MVT VecVT =
Op.getSimpleValueType();
10286 "vector_interleave on non-scalable vector!");
10297 EVT SplitVT = Op0Lo.getValueType();
10300 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10302 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10316 Op.getOperand(0),
Op.getOperand(1));
10343 Concat, EvenIdx, Passthru, Mask, VL);
10345 Concat, OddIdx, Passthru, Mask, VL);
10359 MVT VecVT =
Op.getSimpleValueType();
10362 "vector_interleave on non-scalable vector!");
10375 EVT SplitVT = Op0Lo.getValueType();
10378 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10380 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10402 Op.getOperand(0),
Op.getOperand(1));
10450 MVT VT =
Op.getSimpleValueType();
10455 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10456 if (StepValImm != 1) {
10465 VL, VT,
DL, DAG, Subtarget);
10480 MVT VecVT =
Op.getSimpleValueType();
10490 unsigned MaxVLMAX =
10500 if (MaxVLMAX > 256 && EltSize == 8) {
10547 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10557 MVT VecVT =
Op.getSimpleValueType();
10561 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10562 SDValue DownOffset, UpOffset;
10563 if (ImmValue >= 0) {
10579 DownOffset, TrueMask, UpOffset);
10580 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10586RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10589 auto *
Load = cast<LoadSDNode>(
Op);
10592 Load->getMemoryVT(),
10593 *
Load->getMemOperand()) &&
10594 "Expecting a correctly-aligned load");
10596 MVT VT =
Op.getSimpleValueType();
10602 const auto [MinVLMAX, MaxVLMAX] =
10605 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10619 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10628 Load->getMemoryVT(),
Load->getMemOperand());
10635RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10638 auto *
Store = cast<StoreSDNode>(
Op);
10641 Store->getMemoryVT(),
10642 *
Store->getMemOperand()) &&
10643 "Expecting a correctly-aligned store");
10665 const auto [MinVLMAX, MaxVLMAX] =
10668 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10680 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10683 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10684 Store->getMemoryVT(),
Store->getMemOperand());
10690 MVT VT =
Op.getSimpleValueType();
10692 const auto *MemSD = cast<MemSDNode>(
Op);
10693 EVT MemVT = MemSD->getMemoryVT();
10695 SDValue Chain = MemSD->getChain();
10699 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10700 Mask = VPLoad->getMask();
10702 VL = VPLoad->getVectorLength();
10704 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10705 Mask = MLoad->getMask();
10706 PassThru = MLoad->getPassThru();
10713 MVT ContainerVT = VT;
10727 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10744 Chain =
Result.getValue(1);
10756 const auto *MemSD = cast<MemSDNode>(
Op);
10757 EVT MemVT = MemSD->getMemoryVT();
10759 SDValue Chain = MemSD->getChain();
10763 bool IsCompressingStore =
false;
10764 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10765 Val = VPStore->getValue();
10766 Mask = VPStore->getMask();
10767 VL = VPStore->getVectorLength();
10769 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10770 Val = MStore->getValue();
10771 Mask = MStore->getMask();
10772 IsCompressingStore = MStore->isCompressingStore();
10781 MVT ContainerVT = VT;
10786 if (!IsUnmasked || IsCompressingStore) {
10795 if (IsCompressingStore) {
10798 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10805 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10814 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10818RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10820 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10823 MVT VT =
Op.getSimpleValueType();
10837 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10844 unsigned Opc =
Op.getOpcode();
10851 MVT VT =
Op.getSimpleValueType();
10884 MVT ContainerInVT = InVT;
10903 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10907 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10915 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10920 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10933 MVT VT =
Op.getSimpleValueType();
10937 "Unexpected type for ISD::ABS");
10939 MVT ContainerVT = VT;
10946 if (
Op->getOpcode() == ISD::VP_ABS) {
10947 Mask =
Op->getOperand(1);
10951 VL =
Op->getOperand(2);
10959 DAG.
getUNDEF(ContainerVT), Mask, VL);
10961 DAG.
getUNDEF(ContainerVT), Mask, VL);
10968SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10971 MVT VT =
Op.getSimpleValueType();
10975 "Can only handle COPYSIGN with matching types.");
10984 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10989SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10991 MVT VT =
Op.getSimpleValueType();
10994 MVT I1ContainerVT =
11008 Op2, DAG.
getUNDEF(ContainerVT), VL);
11019 MVT VT =
Op.getSimpleValueType();
11024 for (
const SDValue &V :
Op->op_values()) {
11025 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11028 if (!
V.getValueType().isVector()) {
11034 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11035 "Only fixed length vectors are supported!");
11049 if (
Op->isStrictFPOpcode()) {
11058 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11072 MVT VT =
Op.getSimpleValueType();
11075 MVT ContainerVT = VT;
11081 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11087 if (*MaskIdx == OpIdx.index())
11091 if (
Op.getOpcode() == ISD::VP_MERGE) {
11095 assert(
Op.getOpcode() == ISD::VP_SELECT);
11102 if (!
V.getValueType().isFixedLengthVector()) {
11107 MVT OpVT =
V.getSimpleValueType();
11109 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11110 "Only fixed length vectors are supported!");
11115 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11125 MVT VT =
Op.getSimpleValueType();
11131 MVT ContainerVT = VT;
11141 DAG.
getUNDEF(ContainerVT), Zero, VL);
11144 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11146 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11149 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11158 MVT VT =
Op.getSimpleValueType();
11162 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11166 MVT ContainerVT = VT;
11176 switch (Condition) {
11244 MVT DstVT =
Op.getSimpleValueType();
11245 MVT SrcVT = Src.getSimpleValueType();
11258 if (DstEltSize >= SrcEltSize) {
11267 if (SrcEltSize == 1) {
11278 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11279 }
else if (DstEltSize > (2 * SrcEltSize)) {
11283 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11289 "Wrong input/output vector types");
11292 if (DstEltSize > (2 * SrcEltSize)) {
11308 MVT InterimFVT = DstVT;
11309 if (SrcEltSize > (2 * DstEltSize)) {
11310 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11317 if (InterimFVT != DstVT) {
11323 "Wrong input/output vector types");
11327 if (DstEltSize == 1) {
11330 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11340 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11350 while (InterimIVT != DstVT) {
11362 MVT VT =
Op.getSimpleValueType();
11369RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11381 MVT VT =
Op.getSimpleValueType();
11382 MVT ContainerVT = VT;
11395 if (IsMaskVector) {
11406 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11415 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11418 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11419 SDValue DownOffset, UpOffset;
11420 if (ImmValue >= 0) {
11434 Op1, DownOffset, Mask, UpOffset);
11438 if (IsMaskVector) {
11442 {Result, DAG.getConstant(0, DL, ContainerVT),
11443 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11453RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11456 MVT VT =
Op.getSimpleValueType();
11463 MVT ContainerVT = VT;
11471 MVT GatherVT = ContainerVT;
11475 if (IsMaskVector) {
11486 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11492 unsigned MaxVLMAX =
11501 if (MaxVLMAX > 256 && EltSize == 8) {
11529 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11531 if (IsMaskVector) {
11554 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11556 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11558 DAG.
getUNDEF(GatherVT), Mask, EVL);
11560 if (IsMaskVector) {
11575 MVT VT =
Op.getSimpleValueType();
11577 return lowerVPOp(
Op, DAG);
11584 MVT ContainerVT = VT;
11603 MVT VT =
Op.getSimpleValueType();
11604 MVT ContainerVT = VT;
11610 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11616 : Intrinsic::riscv_vlse_mask,
11619 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11620 VPNode->getStride()};
11628 Ops.
push_back(VPNode->getVectorLength());
11636 VPNode->getMemoryVT(), VPNode->getMemOperand());
11650 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11651 SDValue StoreVal = VPNode->getValue();
11653 MVT ContainerVT = VT;
11664 : Intrinsic::riscv_vsse_mask,
11667 VPNode->getBasePtr(), VPNode->getStride()};
11675 Ops.
push_back(VPNode->getVectorLength());
11678 Ops, VPNode->getMemoryVT(),
11679 VPNode->getMemOperand());
11691 MVT VT =
Op.getSimpleValueType();
11693 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11694 EVT MemVT = MemSD->getMemoryVT();
11696 SDValue Chain = MemSD->getChain();
11702 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11703 Index = VPGN->getIndex();
11704 Mask = VPGN->getMask();
11706 VL = VPGN->getVectorLength();
11711 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11712 Index = MGN->getIndex();
11713 Mask = MGN->getMask();
11714 PassThru = MGN->getPassThru();
11718 MVT IndexVT =
Index.getSimpleValueType();
11722 "Unexpected VTs!");
11723 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11726 "Unexpected extending MGATHER/VP_GATHER");
11732 MVT ContainerVT = VT;
11756 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11773 Chain =
Result.getValue(1);
11790 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11791 EVT MemVT = MemSD->getMemoryVT();
11793 SDValue Chain = MemSD->getChain();
11796 [[maybe_unused]]
bool IsTruncatingStore =
false;
11799 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11800 Index = VPSN->getIndex();
11801 Mask = VPSN->getMask();
11802 Val = VPSN->getValue();
11803 VL = VPSN->getVectorLength();
11805 IsTruncatingStore =
false;
11808 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11809 Index = MSN->getIndex();
11810 Mask = MSN->getMask();
11811 Val = MSN->getValue();
11812 IsTruncatingStore = MSN->isTruncatingStore();
11816 MVT IndexVT =
Index.getSimpleValueType();
11820 "Unexpected VTs!");
11821 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11824 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11830 MVT ContainerVT = VT;
11854 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11864 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11873 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11881 static const int Table =
11905 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11911 static const unsigned Table =
11934 bool isRISCV64 = Subtarget.
is64Bit();
11998 switch (
N->getOpcode()) {
12000 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12006 "Unexpected custom legalisation");
12007 bool IsStrict =
N->isStrictFPOpcode();
12010 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12028 Opc,
DL, VTs, Chain, Op0,
12062 std::tie(Result, Chain) =
12063 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12091 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12102 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12103 "has custom type legalization on riscv32");
12105 SDValue LoCounter, HiCounter;
12109 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12111 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12114 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12116 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12120 N->getOperand(0), LoCounter, HiCounter);
12144 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12145 unsigned XLen = Subtarget.
getXLen();
12148 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12156 if (LHSIsU == RHSIsU)
12173 if (RHSIsU && LHSIsS && !RHSIsS)
12175 else if (LHSIsU && RHSIsS && !LHSIsS)
12185 "Unexpected custom legalisation");
12192 "Unexpected custom legalisation");
12195 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12221 "Unexpected custom legalisation");
12222 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12223 Subtarget.hasVendorXTHeadBb()) &&
12224 "Unexpected custom legalization");
12225 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12226 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12235 "Unexpected custom legalisation");
12249 MVT VT =
N->getSimpleValueType(0);
12250 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12251 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12252 "Unexpected custom legalisation");
12264 if (VT != MVT::i32)
12273 "Unexpected custom legalisation");
12277 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12294 EVT OType =
N->getValueType(1);
12307 "Unexpected custom legalisation");
12324 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12328 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12346 "Unexpected custom legalisation");
12347 if (Subtarget.hasStdExtZbb()) {
12368 "Unexpected custom legalisation");
12374 "Unexpected custom legalisation");
12376 if (Subtarget.hasStdExtZbb()) {
12410 EVT VT =
N->getValueType(0);
12415 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12419 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12420 Subtarget.hasStdExtZfbfmin()) {
12423 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12428 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12430 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12450 MVT VT =
N->getSimpleValueType(0);
12452 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12453 "Unexpected custom legalisation");
12454 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
12480 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12482 "Unexpected EXTRACT_VECTOR_ELT legalization");
12485 MVT ContainerVT = VecVT;
12513 DAG.
getUNDEF(ContainerVT), Mask, VL);
12521 unsigned IntNo =
N->getConstantOperandVal(0);
12525 "Don't know how to custom type legalize this intrinsic!");
12526 case Intrinsic::experimental_get_vector_length: {
12531 case Intrinsic::experimental_cttz_elts: {
12537 case Intrinsic::riscv_orc_b:
12538 case Intrinsic::riscv_brev8:
12539 case Intrinsic::riscv_sha256sig0:
12540 case Intrinsic::riscv_sha256sig1:
12541 case Intrinsic::riscv_sha256sum0:
12542 case Intrinsic::riscv_sha256sum1:
12543 case Intrinsic::riscv_sm3p0:
12544 case Intrinsic::riscv_sm3p1: {
12545 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12565 case Intrinsic::riscv_sm4ks:
12566 case Intrinsic::riscv_sm4ed: {
12574 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12578 case Intrinsic::riscv_mopr: {
12579 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12589 case Intrinsic::riscv_moprr: {
12590 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12602 case Intrinsic::riscv_clmul: {
12603 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12614 case Intrinsic::riscv_clmulh:
12615 case Intrinsic::riscv_clmulr: {
12616 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12644 case Intrinsic::riscv_vmv_x_s: {
12645 EVT VT =
N->getValueType(0);
12647 if (VT.
bitsLT(XLenVT)) {
12656 "Unexpected custom legalization");
12694 case ISD::VP_REDUCE_ADD:
12695 case ISD::VP_REDUCE_AND:
12696 case ISD::VP_REDUCE_OR:
12697 case ISD::VP_REDUCE_XOR:
12698 case ISD::VP_REDUCE_SMAX:
12699 case ISD::VP_REDUCE_UMAX:
12700 case ISD::VP_REDUCE_SMIN:
12701 case ISD::VP_REDUCE_UMIN:
12765 const EVT VT =
N->getValueType(0);
12766 const unsigned Opc =
N->getOpcode();
12773 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12778 "Inconsistent mappings");
12789 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12792 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12807 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12809 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12810 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12814 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12821 if (
LHS.getOpcode() != ReduceOpc)
12836 auto Flags = ReduceVec->
getFlags();
12837 Flags.intersectWith(
N->getFlags());
12838 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12848 auto BinOpToRVVReduce = [](
unsigned Opc) {
12877 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12880 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12883 unsigned Opc =
N->getOpcode();
12884 unsigned ReduceIdx;
12885 if (IsReduction(
N->getOperand(0), Opc))
12887 else if (IsReduction(
N->getOperand(1), Opc))
12893 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12896 SDValue Extract =
N->getOperand(ReduceIdx);
12928 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12955 if (!Subtarget.hasStdExtZba())
12959 EVT VT =
N->getValueType(0);
12971 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12972 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12975 int64_t C0 = N0C->getSExtValue();
12976 int64_t C1 = N1C->getSExtValue();
12977 if (C0 <= 0 || C1 <= 0)
12981 int64_t Bits = std::min(C0, C1);
12982 int64_t Diff = std::abs(C0 - C1);
12983 if (Diff != 1 && Diff != 2 && Diff != 3)
13011 EVT VT =
N->getValueType(0);
13019 if ((!Subtarget.hasStdExtZicond() &&
13020 !Subtarget.hasVendorXVentanaCondOps()) ||
13042 bool SwapSelectOps;
13048 SwapSelectOps =
false;
13049 NonConstantVal = FalseVal;
13051 SwapSelectOps =
true;
13052 NonConstantVal = TrueVal;
13058 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13106 EVT VT =
N->getValueType(0);
13114 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13115 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13121 if (!N0C->hasOneUse())
13123 int64_t C0 = N0C->getSExtValue();
13124 int64_t C1 = N1C->getSExtValue();
13126 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13129 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13130 !isInt<12>(C0 * (C1 / C0))) {
13133 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13134 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13137 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13138 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13165 EVT VT =
N->getValueType(0);
13196 unsigned OuterExtend =
13200 OuterExtend,
SDLoc(
N), VT,
13208 EVT VT =
N->getValueType(0);
13253 EVT VT =
N->getValueType(0);
13257 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13263 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13273 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13295 EVT VT =
N->getValueType(0);
13324 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13348 EVT VT =
N->getValueType(0);
13368 EVT VT =
N->getValueType(0);
13454 EVT VT =
N->getValueType(0);
13525 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13530 const APInt &Imm = ConstN00->getAPIntValue();
13531 if ((Imm + 1).isSignedIntN(12))
13548 LHS.getValueType());
13569 EVT VT =
N->getValueType(0);
13581 if (!Subtarget.hasStdExtZba() && !Subtarget.hasVendorXTHeadBa())
13595 for (
uint64_t Divisor : {3, 5, 9}) {
13596 if (MulAmt % Divisor != 0)
13598 uint64_t MulAmt2 = MulAmt / Divisor;
13605 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
13606 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
13622 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13638 if (ScaleShift >= 1 && ScaleShift < 4) {
13639 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13653 for (
uint64_t Divisor : {3, 5, 9}) {
13658 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13669 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13671 if (ScaleShift >= 1 && ScaleShift < 4) {
13672 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13703 EVT VT =
N->getValueType(0);
13711 unsigned AddSubOpc;
13717 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13718 AddSubOpc = V->getOpcode();
13720 SDValue Opnd = V->getOperand(1);
13721 MulOper = V->getOperand(0);
13730 if (IsAddSubWith1(N0)) {
13732 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13735 if (IsAddSubWith1(N1)) {
13737 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13749 if (isIndexTypeSigned(IndexType))
13752 if (!
N->hasOneUse())
13755 EVT VT =
N.getValueType();
13794 EVT SrcVT = Src.getValueType();
13798 NewElen = std::max(NewElen, 8U);
13821 EVT VT =
N->getValueType(0);
13824 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13828 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13840 if (!isIntEqualitySetCC(
Cond))
13849 const APInt &C1 = N1C->getAPIntValue();
13867 EVT VT =
N->getValueType(0);
13871 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13873 Src.getOperand(0));
13881struct CombineResult;
13883enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13910struct NodeExtensionHelper {
13919 bool SupportsFPExt;
13922 bool EnforceOneUse;
13937 return OrigOperand;
13948 unsigned getExtOpc(ExtKind SupportsExt)
const {
13949 switch (SupportsExt) {
13950 case ExtKind::SExt:
13952 case ExtKind::ZExt:
13954 case ExtKind::FPExt:
13965 std::optional<ExtKind> SupportsExt)
const {
13966 if (!SupportsExt.has_value())
13967 return OrigOperand;
13969 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13973 if (
Source.getValueType() == NarrowVT)
13976 unsigned ExtOpc = getExtOpc(*SupportsExt);
13980 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13987 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14005 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
14011 MVT EltVT = SupportsExt == ExtKind::FPExt
14013 :
MVT::getIntegerVT(NarrowSize);
14015 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
14016 "Trying to extend something we can't represent");
14023 static unsigned getSExtOpcode(
unsigned Opcode) {
14046 static unsigned getZExtOpcode(
unsigned Opcode) {
14072 static unsigned getFPExtOpcode(
unsigned Opcode) {
14089 static unsigned getSUOpcode(
unsigned Opcode) {
14091 "SU is only supported for MUL");
14097 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14117 using CombineToTry = std::function<std::optional<CombineResult>(
14118 SDNode * ,
const NodeExtensionHelper & ,
14123 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14127 unsigned Opc = OrigOperand.
getOpcode();
14131 "Unexpected Opcode");
14144 unsigned ScalarBits =
Op.getValueSizeInBits();
14147 if (ScalarBits < EltBits)
14153 if (NarrowSize < 8)
14157 SupportsSExt =
true;
14161 SupportsZExt =
true;
14163 EnforceOneUse =
false;
14170 SupportsZExt =
false;
14171 SupportsSExt =
false;
14172 SupportsFPExt =
false;
14173 EnforceOneUse =
true;
14174 unsigned Opc = OrigOperand.
getOpcode();
14196 SupportsZExt =
true;
14199 SupportsSExt =
true;
14202 SupportsFPExt =
true;
14206 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14214 static bool isSupportedRoot(
const SDNode *Root,
14243 Subtarget.hasStdExtZvbb();
14245 return Subtarget.hasStdExtZvbb();
14254 assert(isSupportedRoot(Root, Subtarget) &&
14255 "Trying to build an helper with an "
14256 "unsupported root");
14257 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14273 if (OperandIdx == 1) {
14282 EnforceOneUse =
false;
14287 fillUpExtensionSupport(Root, DAG, Subtarget);
14293 static std::pair<SDValue, SDValue>
14296 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14315 switch (
N->getOpcode()) {
14353struct CombineResult {
14355 unsigned TargetOpcode;
14357 std::optional<ExtKind> LHSExt;
14358 std::optional<ExtKind> RHSExt;
14362 NodeExtensionHelper
LHS;
14364 NodeExtensionHelper
RHS;
14366 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14367 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14368 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14369 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14378 std::tie(Mask, VL) =
14379 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14393 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14394 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14409static std::optional<CombineResult>
14410canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14411 const NodeExtensionHelper &RHS,
14414 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14415 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14416 Root, LHS, {ExtKind::ZExt}, RHS,
14418 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14419 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14420 Root, LHS, {ExtKind::SExt}, RHS,
14422 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14423 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14424 Root, LHS, {ExtKind::FPExt}, RHS,
14426 return std::nullopt;
14435static std::optional<CombineResult>
14436canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14439 return canFoldToVWWithSameExtensionImpl(
14440 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14448static std::optional<CombineResult>
14449canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14452 if (
RHS.SupportsFPExt)
14453 return CombineResult(
14454 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14455 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14462 return CombineResult(
14463 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14464 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14466 return CombineResult(
14467 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14468 LHS, std::nullopt, RHS, {ExtKind::SExt});
14469 return std::nullopt;
14476static std::optional<CombineResult>
14477canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14480 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14488static std::optional<CombineResult>
14489canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14492 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14500static std::optional<CombineResult>
14501canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14504 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14512static std::optional<CombineResult>
14513canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14517 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14518 return std::nullopt;
14519 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14520 Root, LHS, {ExtKind::SExt}, RHS,
14525NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14536 Strategies.
push_back(canFoldToVWWithSameExtension);
14541 Strategies.
push_back(canFoldToVWWithSameExtension);
14546 Strategies.
push_back(canFoldToVWWithSameExtension);
14553 Strategies.
push_back(canFoldToVWWithZEXT);
14558 Strategies.
push_back(canFoldToVWWithSEXT);
14563 Strategies.
push_back(canFoldToVWWithZEXT);
14568 Strategies.
push_back(canFoldToVWWithFPEXT);
14597 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14603 Inserted.insert(
N);
14606 while (!Worklist.
empty()) {
14608 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14611 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14612 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14613 auto AppendUsersIfNeeded = [&Worklist,
14614 &Inserted](
const NodeExtensionHelper &
Op) {
14615 if (
Op.needToPromoteOtherUsers()) {
14616 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14617 if (Inserted.insert(TheUse).second)
14629 NodeExtensionHelper::getSupportedFoldings(
N);
14631 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14632 bool Matched =
false;
14633 for (
int Attempt = 0;
14634 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14637 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14638 FoldingStrategies) {
14639 std::optional<CombineResult> Res =
14640 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14647 if (Res->LHSExt.has_value())
14648 AppendUsersIfNeeded(
LHS);
14649 if (Res->RHSExt.has_value())
14650 AppendUsersIfNeeded(
RHS);
14661 SDValue InputRootReplacement;
14668 for (CombineResult Res : CombinesToApply) {
14669 SDValue NewValue = Res.materialize(DAG, Subtarget);
14670 if (!InputRootReplacement) {
14672 "First element is expected to be the current node");
14673 InputRootReplacement = NewValue;
14678 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14682 return InputRootReplacement;
14689 unsigned Opc =
N->getOpcode();
14694 SDValue MergeOp =
N->getOperand(1);
14695 unsigned MergeOpc = MergeOp.
getOpcode();
14706 SDValue Passthru =
N->getOperand(2);
14720 Z = Z.getOperand(1);
14726 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14733 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14762 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14768 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14770 if (MemVT == MVT::i32)
14776 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14811 if (!Subtarget.hasVendorXTHeadMemPair())
14823 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14825 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14826 return {
Ptr->getOperand(0), C1->getZExtValue()};
14830 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14853 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14856 if (Base1 != Base2)
14860 bool Valid =
false;
14861 if (MemVT == MVT::i32) {
14863 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14865 }
else if (MemVT == MVT::i64) {
14867 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14901 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14909 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14919 EVT VT =
N->getValueType(0);
14922 MVT SrcVT = Src.getSimpleValueType();
14923 MVT SrcContainerVT = SrcVT;
14925 SDValue XVal = Src.getOperand(0);
14952 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14956 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14960 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
14973 if (VT != MVT::i32 && VT != XLenVT)
15003 EVT DstVT =
N->getValueType(0);
15004 if (DstVT != XLenVT)
15010 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15018 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15021 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
15030 if (SatVT == DstVT)
15032 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15038 Src = Src.getOperand(0);
15059 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15065 EVT VT =
N->getValueType(0);
15118 unsigned Offset =
N->isTargetStrictFPOpcode();
15125 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15127 V.getOperand(2) == VL) {
15129 V = V.getOperand(0);
15136 bool NegA = invertIfNegative(
A);
15137 bool NegB = invertIfNegative(
B);
15138 bool NegC = invertIfNegative(
C);
15141 if (!NegA && !NegB && !NegC)
15145 if (
N->isTargetStrictFPOpcode())
15147 {N->getOperand(0), A, B, C, Mask, VL});
15157 if (
N->getValueType(0).isScalableVector() &&
15158 N->getValueType(0).getVectorElementType() == MVT::f32 &&
15165 if (
N->isTargetStrictFPOpcode())
15190 switch (
N->getOpcode()) {
15210 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15211 N->getOperand(2), Mask, VL);
15218 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15221 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15223 uint64_t ShAmt =
N->getConstantOperandVal(1);
15234 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15262 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15275 !isa<ConstantSDNode>(U->getOperand(1)) ||
15276 U->getConstantOperandVal(1) > 32)
15331 if (!
Cond.hasOneUse())
15350 EVT VT =
Cond.getValueType();
15395 LHS =
LHS.getOperand(0);
15405 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15409 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15413 RHS =
LHS.getOperand(1);
15414 LHS =
LHS.getOperand(0);
15423 RHS =
LHS.getOperand(1);
15424 LHS =
LHS.getOperand(0);
15440 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15481 bool Commutative =
true;
15482 unsigned Opc = TrueVal.getOpcode();
15490 Commutative =
false;
15498 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15502 if (FalseVal == TrueVal.getOperand(0))
15504 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15509 EVT VT =
N->getValueType(0);
15511 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15517 assert(IdentityOperand &&
"No identity operand!");
15522 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15523 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15544 CountZeroes =
N->getOperand(2);
15545 ValOnZero =
N->getOperand(1);
15547 CountZeroes =
N->getOperand(1);
15548 ValOnZero =
N->getOperand(2);
15567 if (
Cond->getOperand(0) != CountZeroesArgument)
15583 CountZeroes, BitWidthMinusOne);
15593 EVT VT =
N->getValueType(0);
15594 EVT CondVT =
Cond.getValueType();
15602 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15608 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15629 SDValue TrueVal =
N->getOperand(1);
15630 SDValue FalseVal =
N->getOperand(2);
15645 EVT VT =
N->getValueType(0);
15652 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15667 if (
Op.isUndef()) {
15680 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15684 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15685 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15689 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15717 const unsigned InVecOpcode = InVec->
getOpcode();
15727 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15734 InVecLHS, InValLHS, EltNo);
15736 InVecRHS, InValRHS, EltNo);
15745 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15748 unsigned Elt = IndexC->getZExtValue();
15756 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15759 ConcatOp, InVal, NewIdx);
15763 ConcatOps[ConcatOpIdx] = ConcatOp;
15775 EVT VT =
N->getValueType(0);
15785 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15787 !
SDValue(BaseLd, 0).hasOneUse())
15790 EVT BaseLdVT = BaseLd->getValueType(0);
15797 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15798 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15800 Ld->getValueType(0) != BaseLdVT)
15809 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15811 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15816 if (BIO1.equalBaseIndex(BIO2, DAG))
15821 SDValue P2 = Ld2->getBasePtr();
15827 return std::nullopt;
15831 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15836 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15837 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15845 unsigned WideScalarBitWidth =
15858 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15859 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15860 ? std::get<SDValue>(StrideVariant)
15863 if (MustNegateStride)
15876 BaseLd->getBasePtr(), Stride, AllOneMask};
15879 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15880 ConstStride && ConstStride->getSExtValue() >= 0)
15884 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15890 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15894 Ops, WideVecVT, MMO);
15906 if (
N->getValueType(0).isFixedLengthVector())
15909 SDValue Addend =
N->getOperand(0);
15913 SDValue AddMergeOp =
N->getOperand(2);
15918 auto IsVWMulOpc = [](
unsigned Opc) {
15947 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15948 }(
N, DAG, Subtarget);
15953 if (AddMask != MulMask || AddVL != MulVL)
15958 "Unexpected opcode after VWMACC_VL");
15960 "Unexpected opcode after VWMACC_VL!");
15962 "Unexpected opcode after VWMUL_VL!");
15964 "Unexpected opcode after VWMUL_VL!");
15967 EVT VT =
N->getValueType(0);
15983 const EVT IndexVT =
Index.getValueType();
15987 if (!isIndexTypeSigned(IndexType))
16019 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16022 if (
Index->getOperand(i)->isUndef())
16025 if (
C % ElementSize != 0)
16027 C =
C / ElementSize;
16031 ActiveLanes.
set(
C);
16033 return ActiveLanes.
all();
16051 if (NumElems % 2 != 0)
16055 const unsigned WiderElementSize = ElementSize * 2;
16056 if (WiderElementSize > ST.getELen()/8)
16059 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16062 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16065 if (
Index->getOperand(i)->isUndef())
16071 if (
C % WiderElementSize != 0)
16076 if (
C !=
Last + ElementSize)
16092 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16103 switch (
N->getOpcode()) {
16123 APInt V =
C->getValueAPF().bitcastToAPInt();
16158 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16159 SimplifyDemandedLowBitsHelper(1, 5))
16167 if (SimplifyDemandedLowBitsHelper(0, 32))
16184 MVT VT =
N->getSimpleValueType(0);
16193 "Unexpected value type!");
16216 EVT VT =
N->getValueType(0);
16281 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16286 Src.getOperand(0));
16291 Src.getOperand(0), Src.getOperand(1));
16306 auto IsTruncNode = [](
SDValue V) {
16309 SDValue VL = V.getOperand(2);
16310 auto *
C = dyn_cast<ConstantSDNode>(VL);
16312 bool IsVLMAXForVMSET = (
C &&
C->isAllOnes()) ||
16313 (isa<RegisterSDNode>(VL) &&
16314 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16323 while (IsTruncNode(
Op)) {
16324 if (!
Op.hasOneUse())
16326 Op =
Op.getOperand(0);
16358 unsigned Opc =
N->getOpcode();
16373 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16383 N->getValueType(0), Val,
Cond.getOperand(0));
16394 SDValue FalseV =
N->getOperand(4);
16396 EVT VT =
N->getValueType(0);
16399 if (TrueV == FalseV)
16404 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16410 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16411 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16414 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16415 isInt<12>(TrueSImm - FalseSImm)) {
16431 {LHS, RHS, CC, TrueV, FalseV});
16498 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16511 EVT VT =
N->getValueType(0);
16535 const auto *MGN = dyn_cast<MaskedGatherSDNode>(
N);
16536 const EVT VT =
N->getValueType(0);
16538 SDValue ScaleOp = MGN->getScale();
16540 assert(!MGN->isIndexScaled() &&
16541 "Scaled gather/scatter should not be formed");
16546 N->getVTList(), MGN->getMemoryVT(),
DL,
16547 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16548 MGN->getBasePtr(), Index, ScaleOp},
16549 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16553 N->getVTList(), MGN->getMemoryVT(),
DL,
16554 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16555 MGN->getBasePtr(), Index, ScaleOp},
16556 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16562 if (std::optional<VIDSequence> SimpleVID =
16564 SimpleVID && SimpleVID->StepDenominator == 1) {
16565 const int64_t StepNumerator = SimpleVID->StepNumerator;
16566 const int64_t Addend = SimpleVID->Addend;
16573 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16583 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16584 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16586 Ops, VT, MGN->getMemOperand());
16594 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16596 MGN->getMemoryVT(), MGN->getMemOperand(),
16605 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16607 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16609 EVT IndexVT =
Index.getValueType()
16616 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16618 EltCnt.divideCoefficientBy(2));
16621 EltCnt.divideCoefficientBy(2));
16626 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16635 const auto *MSN = dyn_cast<MaskedScatterSDNode>(
N);
16637 SDValue ScaleOp = MSN->getScale();
16639 assert(!MSN->isIndexScaled() &&
16640 "Scaled gather/scatter should not be formed");
16645 N->getVTList(), MSN->getMemoryVT(),
DL,
16646 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16648 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16652 N->getVTList(), MSN->getMemoryVT(),
DL,
16653 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16655 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16657 EVT VT = MSN->getValue()->getValueType(0);
16659 if (!MSN->isTruncatingStore() &&
16663 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16664 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16665 MSN->getMemoryVT(), MSN->getMemOperand(),
16670 case ISD::VP_GATHER: {
16671 const auto *VPGN = dyn_cast<VPGatherSDNode>(
N);
16673 SDValue ScaleOp = VPGN->getScale();
16675 assert(!VPGN->isIndexScaled() &&
16676 "Scaled gather/scatter should not be formed");
16681 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16682 ScaleOp, VPGN->getMask(),
16683 VPGN->getVectorLength()},
16684 VPGN->getMemOperand(), IndexType);
16688 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16689 ScaleOp, VPGN->getMask(),
16690 VPGN->getVectorLength()},
16691 VPGN->getMemOperand(), IndexType);
16695 case ISD::VP_SCATTER: {
16696 const auto *VPSN = dyn_cast<VPScatterSDNode>(
N);
16698 SDValue ScaleOp = VPSN->getScale();
16700 assert(!VPSN->isIndexScaled() &&
16701 "Scaled gather/scatter should not be formed");
16706 {VPSN->getChain(), VPSN->getValue(),
16707 VPSN->getBasePtr(), Index, ScaleOp,
16708 VPSN->getMask(), VPSN->getVectorLength()},
16709 VPSN->getMemOperand(), IndexType);
16713 {VPSN->getChain(), VPSN->getValue(),
16714 VPSN->getBasePtr(), Index, ScaleOp,
16715 VPSN->getMask(), VPSN->getVectorLength()},
16716 VPSN->getMemOperand(), IndexType);
16730 EVT VT =
N->getValueType(0);
16733 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16734 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16752 EVT VT =
N->getValueType(0);
16756 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16786 if (
N->getValueType(0).isScalableVector() &&
16787 N->getValueType(0).getVectorElementType() == MVT::f32 &&
16802 auto *Store = cast<StoreSDNode>(
N);
16803 SDValue Chain = Store->getChain();
16804 EVT MemVT = Store->getMemoryVT();
16805 SDValue Val = Store->getValue();
16808 bool IsScalarizable =
16810 Store->isSimple() &&
16840 NewVT, *Store->getMemOperand())) {
16842 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16843 Store->getPointerInfo(), Store->getOriginalAlign(),
16844 Store->getMemOperand()->getFlags());
16852 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16854 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16856 L->getMemoryVT() == MemVT) {
16859 NewVT, *Store->getMemOperand()) &&
16861 NewVT, *L->getMemOperand())) {
16863 L->getPointerInfo(), L->getOriginalAlign(),
16864 L->getMemOperand()->getFlags());
16865 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
16866 Store->getPointerInfo(), Store->getOriginalAlign(),
16867 Store->getMemOperand()->getFlags());
16879 MVT VecVT = Src.getSimpleValueType();
16886 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
16889 Store->getMemOperand(), Store->getAddressingMode(),
16890 Store->isTruncatingStore(),
false);
16897 EVT VT =
N->getValueType(0);
16919 const MVT VT =
N->getSimpleValueType(0);
16920 SDValue Passthru =
N->getOperand(0);
16921 SDValue Scalar =
N->getOperand(1);
16930 const MVT VT =
N->getSimpleValueType(0);
16931 SDValue Passthru =
N->getOperand(0);
16932 SDValue Scalar =
N->getOperand(1);
16937 unsigned ScalarSize = Scalar.getValueSizeInBits();
16939 if (ScalarSize > EltWidth && Passthru.
isUndef())
16940 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
16947 (!Const || Const->isZero() ||
16948 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
16958 if (
N->getOperand(0).isUndef() &&
16961 Src.getOperand(0).getValueType().isScalableVector()) {
16962 EVT VT =
N->getValueType(0);
16963 EVT SrcVT = Src.getOperand(0).getValueType();
16967 return Src.getOperand(0);
16973 const MVT VT =
N->getSimpleValueType(0);
16974 SDValue Passthru =
N->getOperand(0);
16975 SDValue Scalar =
N->getOperand(1);
16979 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
16980 return Scalar.getOperand(0);
16989 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
16999 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17007 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17009 if (M1VT.
bitsLT(VecVT)) {
17020 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17025 case Intrinsic::riscv_masked_strided_load: {
17026 MVT VT =
N->getSimpleValueType(0);
17027 auto *Load = cast<MemIntrinsicSDNode>(
N);
17028 SDValue PassThru =
N->getOperand(2);
17030 SDValue Stride =
N->getOperand(4);
17036 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17037 StrideC && StrideC->getZExtValue() == ElementSize)
17039 DAG.
getUNDEF(XLenVT), Mask, PassThru,
17040 Load->getMemoryVT(), Load->getMemOperand(),
17044 case Intrinsic::riscv_masked_strided_store: {
17045 auto *Store = cast<MemIntrinsicSDNode>(
N);
17048 SDValue Stride =
N->getOperand(4);
17053 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
17054 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
17055 StrideC && StrideC->getZExtValue() == ElementSize)
17058 Value.getValueType(), Store->getMemOperand(),
17062 case Intrinsic::riscv_vcpop:
17063 case Intrinsic::riscv_vcpop_mask:
17064 case Intrinsic::riscv_vfirst:
17065 case Intrinsic::riscv_vfirst_mask: {
17067 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17068 IntNo == Intrinsic::riscv_vfirst_mask)
17069 VL =
N->getOperand(3);
17074 EVT VT =
N->getValueType(0);
17075 if (IntNo == Intrinsic::riscv_vfirst ||
17076 IntNo == Intrinsic::riscv_vfirst_mask)
17085 EVT VT =
N->getValueType(0);
17089 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17108 EVT XVT,
unsigned KeptBits)
const {
17113 if (XVT != MVT::i32 && XVT != MVT::i64)
17117 if (KeptBits == 32 || KeptBits == 64)
17121 return Subtarget.hasStdExtZbb() &&
17122 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17130 "Expected shift op");
17141 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17142 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17144 const APInt &C1Int = C1->getAPIntValue();
17145 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17171 if (C1Cost < ShiftedC1Cost)
17185 EVT VT =
Op.getValueType();
17189 unsigned Opcode =
Op.getOpcode();
17197 const APInt &Mask =
C->getAPIntValue();
17206 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17207 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17209 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17210 if (NewMask == Mask)
17215 Op.getOperand(0), NewC);
17228 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17229 if (IsLegalMask(NewMask))
17230 return UseMask(NewMask);
17233 if (VT == MVT::i64) {
17235 if (IsLegalMask(NewMask))
17236 return UseMask(NewMask);
17251 APInt NewMask = ShrunkMask;
17252 if (MinSignedBits <= 12)
17254 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17260 assert(IsLegalMask(NewMask));
17261 return UseMask(NewMask);
17265 static const uint64_t GREVMasks[] = {
17266 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17267 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17269 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17270 unsigned Shift = 1 << Stage;
17271 if (ShAmt & Shift) {
17273 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17285 const APInt &DemandedElts,
17287 unsigned Depth)
const {
17289 unsigned Opc =
Op.getOpcode();
17294 "Should use MaskedValueIsZero if you don't know whether Op"
17295 " is a target node!");
17378 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17381 if (MaxVLenB == MinVLenB)
17398 case Intrinsic::riscv_vsetvli:
17399 case Intrinsic::riscv_vsetvlimax: {
17400 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17401 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17407 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17410 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17411 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17413 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17426 unsigned Depth)
const {
17427 switch (
Op.getOpcode()) {
17433 if (Tmp == 1)
return 1;
17436 return std::min(Tmp, Tmp2);
17448 if (Tmp < 33)
return 1;
17473 unsigned XLen = Subtarget.
getXLen();
17474 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17475 if (EltBits <= XLen)
17476 return XLen - EltBits + 1;
17480 unsigned IntNo =
Op.getConstantOperandVal(1);
17484 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17485 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17486 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17487 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17488 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17489 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17490 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17491 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17492 case Intrinsic::riscv_masked_cmpxchg_i64:
17500 assert(Subtarget.hasStdExtA());
17515 switch (
Op.getOpcode()) {
17521 return !
Op.getValueType().isInteger();
17529 assert(Ld &&
"Unexpected null LoadSDNode");
17537 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17538 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17539 CNode->getOffset() != 0)
17547 auto *CNode = GetSupportedConstantPool(
Ptr);
17548 if (!CNode || CNode->getTargetFlags() != 0)
17551 return CNode->getConstVal();
17559 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17560 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17566 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17569 return CNodeLo->getConstVal();
17574 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17606 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17609 int64_t LoCounter =
MI.getOperand(2).getImm();
17610 int64_t HiCounter =
MI.getOperand(3).getImm();
17620 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17632 MI.eraseFromParent();
17640 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17648 Register SrcReg =
MI.getOperand(2).getReg();
17668 MI.eraseFromParent();
17675 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17676 "Unexpected instruction");
17682 Register DstReg =
MI.getOperand(0).getReg();
17705 MI.eraseFromParent();
17710 switch (
MI.getOpcode()) {
17713 case RISCV::Select_GPR_Using_CC_GPR:
17714 case RISCV::Select_GPR_Using_CC_Imm:
17715 case RISCV::Select_FPR16_Using_CC_GPR:
17716 case RISCV::Select_FPR16INX_Using_CC_GPR:
17717 case RISCV::Select_FPR32_Using_CC_GPR:
17718 case RISCV::Select_FPR32INX_Using_CC_GPR:
17719 case RISCV::Select_FPR64_Using_CC_GPR:
17720 case RISCV::Select_FPR64INX_Using_CC_GPR:
17721 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17727 unsigned RelOpcode,
unsigned EqOpcode,
17730 Register DstReg =
MI.getOperand(0).getReg();
17731 Register Src1Reg =
MI.getOperand(1).getReg();
17732 Register Src2Reg =
MI.getOperand(2).getReg();
17734 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17758 MI.eraseFromParent();
17809 F->insert(It, FirstMBB);
17810 F->insert(It, SecondMBB);
17811 F->insert(It, SinkMBB);
17860 First.eraseFromParent();
17899 if (
MI.getOperand(2).isReg())
17900 RHS =
MI.getOperand(2).getReg();
17905 SelectDests.
insert(
MI.getOperand(0).getReg());
17909 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
17910 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
17911 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
17912 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
17913 Next->getOperand(5).isKill()) {
17918 SequenceMBBI != E; ++SequenceMBBI) {
17919 if (SequenceMBBI->isDebugInstr())
17922 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
17923 !SequenceMBBI->getOperand(2).isReg() ||
17924 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
17925 SequenceMBBI->getOperand(3).getImm() !=
CC ||
17926 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
17927 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
17929 LastSelectPseudo = &*SequenceMBBI;
17931 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
17934 if (SequenceMBBI->hasUnmodeledSideEffects() ||
17935 SequenceMBBI->mayLoadOrStore() ||
17936 SequenceMBBI->usesCustomInsertionHook())
17939 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
17954 F->insert(
I, IfFalseMBB);
17955 F->insert(
I, TailMBB);
17959 TailMBB->
push_back(DebugInstr->removeFromParent());
17963 TailMBB->
splice(TailMBB->
end(), HeadMBB,
17973 if (
MI.getOperand(2).isImm())
17976 .
addImm(
MI.getOperand(2).getImm())
17988 auto SelectMBBI =
MI.getIterator();
17989 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
17990 auto InsertionPoint = TailMBB->
begin();
17991 while (SelectMBBI != SelectEnd) {
17992 auto Next = std::next(SelectMBBI);
17995 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
17996 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
17997 .
addReg(SelectMBBI->getOperand(4).getReg())
17999 .
addReg(SelectMBBI->getOperand(5).getReg())
18014 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18015 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18017 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18018 assert(Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18024 unsigned CVTXOpc) {
18030 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18043 .
add(
MI.getOperand(1))
18044 .
add(
MI.getOperand(2))
18045 .
add(
MI.getOperand(3))
18047 .
add(
MI.getOperand(4))
18048 .
add(
MI.getOperand(5))
18049 .
add(
MI.getOperand(6))
18064 .
add(
MI.getOperand(0))
18065 .
add(
MI.getOperand(1))
18067 .
add(
MI.getOperand(3))
18069 .
add(
MI.getOperand(4))
18070 .
add(
MI.getOperand(5))
18071 .
add(
MI.getOperand(6))
18081 MI.eraseFromParent();
18087 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18089 switch (
MI.getOpcode()) {
18092 case RISCV::PseudoFROUND_H:
18093 CmpOpc = RISCV::FLT_H;
18094 F2IOpc = RISCV::FCVT_W_H;
18095 I2FOpc = RISCV::FCVT_H_W;
18096 FSGNJOpc = RISCV::FSGNJ_H;
18097 FSGNJXOpc = RISCV::FSGNJX_H;
18098 RC = &RISCV::FPR16RegClass;
18100 case RISCV::PseudoFROUND_H_INX:
18101 CmpOpc = RISCV::FLT_H_INX;
18102 F2IOpc = RISCV::FCVT_W_H_INX;
18103 I2FOpc = RISCV::FCVT_H_W_INX;
18104 FSGNJOpc = RISCV::FSGNJ_H_INX;
18105 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18106 RC = &RISCV::GPRF16RegClass;
18108 case RISCV::PseudoFROUND_S:
18109 CmpOpc = RISCV::FLT_S;
18110 F2IOpc = RISCV::FCVT_W_S;
18111 I2FOpc = RISCV::FCVT_S_W;
18112 FSGNJOpc = RISCV::FSGNJ_S;
18113 FSGNJXOpc = RISCV::FSGNJX_S;
18114 RC = &RISCV::FPR32RegClass;
18116 case RISCV::PseudoFROUND_S_INX:
18117 CmpOpc = RISCV::FLT_S_INX;
18118 F2IOpc = RISCV::FCVT_W_S_INX;
18119 I2FOpc = RISCV::FCVT_S_W_INX;
18120 FSGNJOpc = RISCV::FSGNJ_S_INX;
18121 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18122 RC = &RISCV::GPRF32RegClass;
18124 case RISCV::PseudoFROUND_D:
18126 CmpOpc = RISCV::FLT_D;
18127 F2IOpc = RISCV::FCVT_L_D;
18128 I2FOpc = RISCV::FCVT_D_L;
18129 FSGNJOpc = RISCV::FSGNJ_D;
18130 FSGNJXOpc = RISCV::FSGNJX_D;
18131 RC = &RISCV::FPR64RegClass;
18133 case RISCV::PseudoFROUND_D_INX:
18135 CmpOpc = RISCV::FLT_D_INX;
18136 F2IOpc = RISCV::FCVT_L_D_INX;
18137 I2FOpc = RISCV::FCVT_D_L_INX;
18138 FSGNJOpc = RISCV::FSGNJ_D_INX;
18139 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18140 RC = &RISCV::GPRRegClass;
18152 F->insert(
I, CvtMBB);
18153 F->insert(
I, DoneMBB);
18164 Register DstReg =
MI.getOperand(0).getReg();
18165 Register SrcReg =
MI.getOperand(1).getReg();
18166 Register MaxReg =
MI.getOperand(2).getReg();
18167 int64_t FRM =
MI.getOperand(3).getImm();
18172 Register FabsReg =
MRI.createVirtualRegister(RC);
18176 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18191 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18213 MI.eraseFromParent();
18220 switch (
MI.getOpcode()) {
18223 case RISCV::ReadCounterWide:
18225 "ReadCounterWide is only to be used on riscv32");
18227 case RISCV::Select_GPR_Using_CC_GPR:
18228 case RISCV::Select_GPR_Using_CC_Imm:
18229 case RISCV::Select_FPR16_Using_CC_GPR:
18230 case RISCV::Select_FPR16INX_Using_CC_GPR:
18231 case RISCV::Select_FPR32_Using_CC_GPR:
18232 case RISCV::Select_FPR32INX_Using_CC_GPR:
18233 case RISCV::Select_FPR64_Using_CC_GPR:
18234 case RISCV::Select_FPR64INX_Using_CC_GPR:
18235 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18237 case RISCV::BuildPairF64Pseudo:
18239 case RISCV::SplitF64Pseudo:
18241 case RISCV::PseudoQuietFLE_H:
18243 case RISCV::PseudoQuietFLE_H_INX:
18244 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18245 case RISCV::PseudoQuietFLT_H:
18247 case RISCV::PseudoQuietFLT_H_INX:
18248 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18249 case RISCV::PseudoQuietFLE_S:
18251 case RISCV::PseudoQuietFLE_S_INX:
18252 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18253 case RISCV::PseudoQuietFLT_S:
18255 case RISCV::PseudoQuietFLT_S_INX:
18256 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18257 case RISCV::PseudoQuietFLE_D:
18259 case RISCV::PseudoQuietFLE_D_INX:
18260 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18261 case RISCV::PseudoQuietFLE_D_IN32X:
18264 case RISCV::PseudoQuietFLT_D:
18266 case RISCV::PseudoQuietFLT_D_INX:
18267 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18268 case RISCV::PseudoQuietFLT_D_IN32X:
18272 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18274 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18276 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18278 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18280 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18282 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18284 case RISCV::PseudoFROUND_H:
18285 case RISCV::PseudoFROUND_H_INX:
18286 case RISCV::PseudoFROUND_S:
18287 case RISCV::PseudoFROUND_S_INX:
18288 case RISCV::PseudoFROUND_D:
18289 case RISCV::PseudoFROUND_D_INX:
18290 case RISCV::PseudoFROUND_D_IN32X:
18292 case TargetOpcode::STATEPOINT:
18298 MI.addOperand(*
MI.getMF(),
18304 case TargetOpcode::STACKMAP:
18305 case TargetOpcode::PATCHPOINT:
18308 "supported on 64-bit targets");
18326 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18358 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18359 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18362 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18363 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18366 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18367 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18371 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18372 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18373 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18375 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18376 RISCV::V20M2, RISCV::V22M2};
18384 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18385 RISCV::X13, RISCV::X14, RISCV::X15,
18386 RISCV::X16, RISCV::X17};
18388 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18389 RISCV::X13, RISCV::X14, RISCV::X15};
18400 static const MCPhysReg FastCCIGPRs[] = {
18401 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18402 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18403 RISCV::X29, RISCV::X30, RISCV::X31};
18406 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18407 RISCV::X13, RISCV::X14, RISCV::X15,
18422 unsigned XLenInBytes = XLen / 8;
18435 Align StackAlign(XLenInBytes);
18436 if (!
EABI || XLen != 32)
18468 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18469 assert(XLen == 32 || XLen == 64);
18470 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18474 if (ArgFlags.
isNest()) {
18475 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18483 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18488 bool UseGPRForF16_F32 =
true;
18491 bool UseGPRForF64 =
true;
18503 UseGPRForF16_F32 = !IsFixed;
18507 UseGPRForF16_F32 = !IsFixed;
18508 UseGPRForF64 = !IsFixed;
18514 UseGPRForF16_F32 =
true;
18515 UseGPRForF64 =
true;
18522 if (UseGPRForF16_F32 &&
18523 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18526 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18543 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18545 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18549 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18558 "PendingLocs and PendingArgFlags out of sync");
18562 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18563 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18611 PendingLocs.
size() <= 2) {
18612 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18617 PendingLocs.
clear();
18618 PendingArgFlags.
clear();
18620 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18626 unsigned StoreSizeBytes = XLen / 8;
18629 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18631 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18633 else if (ValVT == MVT::f64 && !UseGPRForF64)
18669 if (!PendingLocs.
empty()) {
18671 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18673 for (
auto &It : PendingLocs) {
18675 It.convertToReg(Reg);
18680 PendingLocs.clear();
18681 PendingArgFlags.
clear();
18685 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18687 "Expected an XLenVT or vector types at this stage");
18705template <
typename ArgTy>
18707 for (
const auto &ArgIdx :
enumerate(Args)) {
18708 MVT ArgVT = ArgIdx.value().VT;
18710 return ArgIdx.index();
18712 return std::nullopt;
18715void RISCVTargetLowering::analyzeInputArgs(
18718 RISCVCCAssignFn Fn)
const {
18719 unsigned NumArgs =
Ins.size();
18732 for (
unsigned i = 0; i != NumArgs; ++i) {
18736 Type *ArgTy =
nullptr;
18738 ArgTy = FType->getReturnType();
18739 else if (Ins[i].isOrigArg())
18740 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18744 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18746 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18753void RISCVTargetLowering::analyzeOutputArgs(
18756 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18757 unsigned NumArgs = Outs.
size();
18767 for (
unsigned i = 0; i != NumArgs; i++) {
18768 MVT ArgVT = Outs[i].VT;
18770 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18774 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18776 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18830 if (In.isOrigArg()) {
18835 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18836 (
BitWidth < 32 && In.Flags.isZExt())) {
18865 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
18911 ExtType,
DL, LocVT, Chain, FIN,
18928 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18941 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18951 unsigned ValNo,
MVT ValVT,
MVT LocVT,
18954 bool IsFixed,
bool IsRet,
Type *OrigTy,
18957 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18966 if (LocVT == MVT::f16 &&
18967 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
18969 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
18970 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
18971 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
18972 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
18973 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
18979 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18981 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
18982 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
18983 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
18984 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
18985 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18991 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18993 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
18994 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
18995 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
18996 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
18997 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19004 if ((LocVT == MVT::f16 &&
19005 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
19006 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19007 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
19008 Subtarget.hasStdExtZdinx())) {
19015 if (LocVT == MVT::f16) {
19021 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
19027 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
19035 if (AllocatedVReg) {
19070 if (ArgFlags.
isNest()) {
19072 "Attribute 'nest' is not supported in GHC calling convention");
19076 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
19077 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
19079 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19091 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19094 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
19095 RISCV::F18_F, RISCV::F19_F,
19096 RISCV::F20_F, RISCV::F21_F};
19097 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19103 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19106 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19107 RISCV::F24_D, RISCV::F25_D,
19108 RISCV::F26_D, RISCV::F27_D};
19109 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19115 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19116 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19136 switch (CallConv) {
19146 if (Subtarget.hasStdExtE())
19150 "(Zdinx/D) instruction set extensions");
19154 if (Func.hasFnAttribute(
"interrupt")) {
19155 if (!Func.arg_empty())
19157 "Functions with the interrupt attribute cannot have arguments!");
19162 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19164 "Function interrupt attribute argument not supported!");
19169 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19171 std::vector<SDValue> OutChains;
19180 analyzeInputArgs(MF, CCInfo, Ins,
false,
19184 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19205 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19206 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19208 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19210 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19239 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19244 if (VarArgsSaveSize == 0) {
19248 int VaArgOffset = -VarArgsSaveSize;
19256 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19257 VarArgsSaveSize += XLenInBytes;
19264 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19269 Chain,
DL, ArgValue, FIN,
19271 OutChains.push_back(Store);
19285 if (!OutChains.empty()) {
19286 OutChains.push_back(Chain);
19296bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19300 auto CalleeCC = CLI.CallConv;
19301 auto &Outs = CLI.Outs;
19303 auto CallerCC = Caller.getCallingConv();
19310 if (Caller.hasFnAttribute(
"interrupt"))
19325 for (
auto &VA : ArgLocs)
19331 auto IsCallerStructRet = Caller.hasStructRetAttr();
19332 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19333 if (IsCallerStructRet || IsCalleeStructRet)
19338 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19339 if (CalleeCC != CallerCC) {
19340 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19341 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19348 for (
auto &Arg : Outs)
19349 if (Arg.Flags.isByVal())
19384 if (Subtarget.hasStdExtE())
19388 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19394 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19400 "site marked musttail");
19407 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19409 if (!Flags.isByVal())
19413 unsigned Size = Flags.getByValSize();
19414 Align Alignment = Flags.getNonZeroByValAlign();
19421 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19435 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19438 SDValue ArgValue = OutVals[OutIdx];
19458 if (!StackPtr.getNode())
19469 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19487 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19488 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19494 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19495 SDValue PartValue = OutVals[OutIdx + 1];
19496 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19508 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19510 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19512 for (
const auto &Part : Parts) {
19513 SDValue PartValue = Part.first;
19514 SDValue PartOffset = Part.second;
19521 ArgValue = SpillSlot;
19527 if (Flags.isByVal())
19528 ArgValue = ByValArgs[j++];
19535 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19536 "for passing parameters");
19539 if (!StackPtr.getNode())
19552 if (!MemOpChains.
empty())
19558 for (
auto &Reg : RegsToPass) {
19559 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19566 validateCCReservedRegs(RegsToPass, MF);
19571 "Return address register required, but has been reserved."});
19590 for (
auto &Reg : RegsToPass)
19596 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19597 assert(Mask &&
"Missing call preserved mask for calling convention");
19606 "Unexpected CFI type for a direct call");
19636 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19637 auto &VA = RVLocs[i];
19645 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19646 assert(VA.needsCustom());
19671 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19672 MVT VT = Outs[i].VT;
19676 ArgFlags, CCInfo,
true,
true,
19677 nullptr, *
this, Dispatcher))
19709 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19710 SDValue Val = OutVals[OutIdx];
19719 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19723 Register RegHi = RVLocs[++i].getLocReg();
19729 "Return value register required, but has been reserved."});
19745 "Return value register required, but has been reserved."});
19767 if (Func.hasFnAttribute(
"interrupt")) {
19768 if (!Func.getReturnType()->isVoidTy())
19770 "Functions with the interrupt attribute must have void return type!");
19776 if (Kind ==
"supervisor")
19782 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19785void RISCVTargetLowering::validateCCReservedRegs(
19786 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19795 F,
"Argument register required, but has been reserved."});
19801 if (
N->getNumValues() != 1)
19803 if (!
N->hasNUsesOfValue(1, 0))
19806 SDNode *Copy = *
N->use_begin();
19820 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19824 bool HasRet =
false;
19825 for (
SDNode *Node : Copy->uses()) {
19833 Chain = Copy->getOperand(0);
19842#define NODE_NAME_CASE(NODE) \
19843 case RISCVISD::NODE: \
19844 return "RISCVISD::" #NODE;
20097#undef NODE_NAME_CASE
20104 if (Constraint.
size() == 1) {
20105 switch (Constraint[0]) {
20121 if (Constraint ==
"vr" || Constraint ==
"vm")
20127std::pair<unsigned, const TargetRegisterClass *>
20133 if (Constraint.
size() == 1) {
20134 switch (Constraint[0]) {
20139 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20140 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20141 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20142 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20143 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20144 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20145 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20147 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20148 return std::make_pair(0U, &RISCV::FPR16RegClass);
20149 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20150 return std::make_pair(0U, &RISCV::FPR32RegClass);
20151 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20152 return std::make_pair(0U, &RISCV::FPR64RegClass);
20157 }
else if (Constraint ==
"vr") {
20158 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20159 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20161 return std::make_pair(0U, RC);
20163 }
else if (Constraint ==
"vm") {
20164 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20165 return std::make_pair(0U, &RISCV::VMV0RegClass);
20173 .
Case(
"{zero}", RISCV::X0)
20174 .
Case(
"{ra}", RISCV::X1)
20175 .
Case(
"{sp}", RISCV::X2)
20176 .
Case(
"{gp}", RISCV::X3)
20177 .
Case(
"{tp}", RISCV::X4)
20178 .
Case(
"{t0}", RISCV::X5)
20179 .
Case(
"{t1}", RISCV::X6)
20180 .
Case(
"{t2}", RISCV::X7)
20181 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20182 .
Case(
"{s1}", RISCV::X9)
20183 .
Case(
"{a0}", RISCV::X10)
20184 .
Case(
"{a1}", RISCV::X11)
20185 .
Case(
"{a2}", RISCV::X12)
20186 .
Case(
"{a3}", RISCV::X13)
20187 .
Case(
"{a4}", RISCV::X14)
20188 .
Case(
"{a5}", RISCV::X15)
20189 .
Case(
"{a6}", RISCV::X16)
20190 .
Case(
"{a7}", RISCV::X17)
20191 .
Case(
"{s2}", RISCV::X18)
20192 .
Case(
"{s3}", RISCV::X19)
20193 .
Case(
"{s4}", RISCV::X20)
20194 .
Case(
"{s5}", RISCV::X21)
20195 .
Case(
"{s6}", RISCV::X22)
20196 .
Case(
"{s7}", RISCV::X23)
20197 .
Case(
"{s8}", RISCV::X24)
20198 .
Case(
"{s9}", RISCV::X25)
20199 .
Case(
"{s10}", RISCV::X26)
20200 .
Case(
"{s11}", RISCV::X27)
20201 .
Case(
"{t3}", RISCV::X28)
20202 .
Case(
"{t4}", RISCV::X29)
20203 .
Case(
"{t5}", RISCV::X30)
20204 .
Case(
"{t6}", RISCV::X31)
20206 if (XRegFromAlias != RISCV::NoRegister)
20207 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20216 if (Subtarget.hasStdExtF()) {
20218 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20219 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20220 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20221 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20222 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20223 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20224 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20225 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20226 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20227 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20228 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20229 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20230 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20231 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20232 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20233 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20234 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20235 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20236 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20237 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20238 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20239 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20240 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20241 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20242 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20243 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20244 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20245 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20246 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20247 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20248 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20249 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20251 if (FReg != RISCV::NoRegister) {
20252 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20253 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20254 unsigned RegNo = FReg - RISCV::F0_F;
20255 unsigned DReg = RISCV::F0_D + RegNo;
20256 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20258 if (VT == MVT::f32 || VT == MVT::Other)
20259 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20260 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20261 unsigned RegNo = FReg - RISCV::F0_F;
20262 unsigned HReg = RISCV::F0_H + RegNo;
20263 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20270 .
Case(
"{v0}", RISCV::V0)
20271 .
Case(
"{v1}", RISCV::V1)
20272 .
Case(
"{v2}", RISCV::V2)
20273 .
Case(
"{v3}", RISCV::V3)
20274 .
Case(
"{v4}", RISCV::V4)
20275 .
Case(
"{v5}", RISCV::V5)
20276 .
Case(
"{v6}", RISCV::V6)
20277 .
Case(
"{v7}", RISCV::V7)
20278 .
Case(
"{v8}", RISCV::V8)
20279 .
Case(
"{v9}", RISCV::V9)
20280 .
Case(
"{v10}", RISCV::V10)
20281 .
Case(
"{v11}", RISCV::V11)
20282 .
Case(
"{v12}", RISCV::V12)
20283 .
Case(
"{v13}", RISCV::V13)
20284 .
Case(
"{v14}", RISCV::V14)
20285 .
Case(
"{v15}", RISCV::V15)
20286 .
Case(
"{v16}", RISCV::V16)
20287 .
Case(
"{v17}", RISCV::V17)
20288 .
Case(
"{v18}", RISCV::V18)
20289 .
Case(
"{v19}", RISCV::V19)
20290 .
Case(
"{v20}", RISCV::V20)
20291 .
Case(
"{v21}", RISCV::V21)
20292 .
Case(
"{v22}", RISCV::V22)
20293 .
Case(
"{v23}", RISCV::V23)
20294 .
Case(
"{v24}", RISCV::V24)
20295 .
Case(
"{v25}", RISCV::V25)
20296 .
Case(
"{v26}", RISCV::V26)
20297 .
Case(
"{v27}", RISCV::V27)
20298 .
Case(
"{v28}", RISCV::V28)
20299 .
Case(
"{v29}", RISCV::V29)
20300 .
Case(
"{v30}", RISCV::V30)
20301 .
Case(
"{v31}", RISCV::V31)
20303 if (VReg != RISCV::NoRegister) {
20304 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20305 return std::make_pair(VReg, &RISCV::VMRegClass);
20306 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20307 return std::make_pair(VReg, &RISCV::VRRegClass);
20308 for (
const auto *RC :
20309 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20310 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20311 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20312 return std::make_pair(VReg, RC);
20318 std::pair<Register, const TargetRegisterClass *> Res =
20324 if (Res.second == &RISCV::GPRF16RegClass ||
20325 Res.second == &RISCV::GPRF32RegClass ||
20326 Res.second == &RISCV::GPRPairRegClass)
20327 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20335 if (ConstraintCode.
size() == 1) {
20336 switch (ConstraintCode[0]) {
20351 if (Constraint.
size() == 1) {
20352 switch (Constraint[0]) {
20355 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20357 if (isInt<12>(CVal))
20370 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20372 if (isUInt<5>(CVal))
20390 if (Subtarget.hasStdExtZtso()) {
20406 if (Subtarget.hasStdExtZtso()) {
20414 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) &&
20431 if (Subtarget.hasForcedAtomics())
20436 if (Subtarget.hasStdExtZacas() &&
20437 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20443 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20456 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20458 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20460 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20462 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20464 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20466 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20468 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20470 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20479 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20481 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20483 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20485 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20487 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20489 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20491 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20493 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20519 unsigned XLen = Subtarget.
getXLen();
20543 unsigned ValWidth =
20548 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20551 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20563 if (Subtarget.hasForcedAtomics())
20567 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20576 unsigned XLen = Subtarget.
getXLen();
20578 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20583 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20589 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20596 EVT DataVT)
const {
20612 return Subtarget.hasStdExtZfhmin();
20614 return Subtarget.hasStdExtF();
20616 return Subtarget.hasStdExtD();
20648 "RVVBitsPerBlock changed, audit needed");
20657 if (!Subtarget.hasVendorXTHeadMemIdx())
20663 Base =
Op->getOperand(0);
20665 int64_t RHSC =
RHS->getSExtValue();
20671 bool isLegalIndexedOffset =
false;
20672 for (
unsigned i = 0; i < 4; i++)
20673 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20674 isLegalIndexedOffset =
true;
20678 if (!isLegalIndexedOffset)
20695 VT = LD->getMemoryVT();
20696 Ptr = LD->getBasePtr();
20697 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20698 VT = ST->getMemoryVT();
20699 Ptr = ST->getBasePtr();
20718 VT = LD->getMemoryVT();
20719 Ptr = LD->getBasePtr();
20720 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20721 VT = ST->getMemoryVT();
20722 Ptr = ST->getBasePtr();
20765 const Constant *PersonalityFn)
const {
20770 const Constant *PersonalityFn)
const {
20794 const bool HasExtMOrZmmul =
20795 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
20804 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
20806 const APInt &Imm = ConstNode->getAPIntValue();
20807 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20808 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20812 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20813 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20814 (Imm - 8).isPowerOf2()))
20819 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20820 ConstNode->hasOneUse()) {
20821 APInt ImmS = Imm.ashr(Imm.countr_zero());
20822 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20823 (1 - ImmS).isPowerOf2())
20847 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20856 unsigned *
Fast)
const {
20859 *
Fast = Subtarget.enableUnalignedScalarMem();
20860 return Subtarget.enableUnalignedScalarMem();
20876 *
Fast = Subtarget.enableUnalignedVectorMem();
20877 return Subtarget.enableUnalignedVectorMem();
20886 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
20898 if (
Op.size() < MinVLenInBytes)
20908 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
20912 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
20914 if (
Op.isFixedDstAlign())
20915 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
20917 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
20925 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
20926 bool IsABIRegCopy =
CC.has_value();
20928 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20929 PartVT == MVT::f32) {
20947 if (PartVTBitSize % ValueVTBitSize == 0) {
20948 assert(PartVTBitSize >= ValueVTBitSize);
20955 if (ValueEltVT != PartEltVT) {
20956 if (PartVTBitSize > ValueVTBitSize) {
20958 assert(Count != 0 &&
"The number of element should not be zero.");
20959 EVT SameEltTypeVT =
20980 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
20981 bool IsABIRegCopy =
CC.has_value();
20982 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20983 PartVT == MVT::f32) {
21000 if (PartVTBitSize % ValueVTBitSize == 0) {
21001 assert(PartVTBitSize >= ValueVTBitSize);
21002 EVT SameEltTypeVT = ValueVT;
21009 if (ValueEltVT != PartEltVT) {
21011 assert(Count != 0 &&
"The number of element should not be zero.");
21028 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21035 unsigned Opc =
N->getOpcode();
21059 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21073 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21078 if (FVTy->getNumElements() < 2)
21093 return Factor * LMUL <= 8;
21097 Align Alignment)
const {
21109 if (!Subtarget.enableUnalignedVectorMem() &&
21117 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21118 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21119 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21120 Intrinsic::riscv_seg8_load};
21139 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21149 {VTy, LI->getPointerOperandType(), XLenTy});
21151 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21156 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21158 Shuffles[i]->replaceAllUsesWith(SubVec);
21165 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21166 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21167 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21168 Intrinsic::riscv_seg8_store};
21188 unsigned Factor)
const {
21190 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21193 ShuffleVTy->getNumElements() / Factor);
21195 SI->getPointerAddressSpace(),
21196 SI->getModule()->getDataLayout()))
21203 {VTy, SI->getPointerOperandType(), XLenTy});
21208 for (
unsigned i = 0; i < Factor; i++) {
21217 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21218 Ops.
append({SI->getPointerOperand(), VL});
21234 unsigned Factor = 2;
21249 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21252 {ResVTy, LI->getPointerOperandType(), XLenTy});
21253 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21256 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21257 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21258 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21259 Intrinsic::riscv_vlseg8};
21284 unsigned Factor = 2;
21290 SI->getPointerAddressSpace(),
21291 SI->getModule()->getDataLayout()))
21298 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21301 {InVTy, SI->getPointerOperandType(), XLenTy});
21302 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21305 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21306 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21307 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21308 Intrinsic::riscv_vsseg8};
21316 SI->getPointerOperand(), VL});
21326 "Invalid call instruction for a KCFI check");
21328 MBBI->getOpcode()));
21331 Target.setIsRenamable(
false);
21339#define GET_REGISTER_MATCHER
21340#include "RISCVGenAsmMatcher.inc"
21346 if (Reg == RISCV::NoRegister)
21348 if (Reg == RISCV::NoRegister)
21360 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21362 if (NontemporalInfo ==
nullptr)
21370 int NontemporalLevel = 5;
21371 const MDNode *RISCVNontemporalInfo =
21372 I.getMetadata(
"riscv-nontemporal-domain");
21373 if (RISCVNontemporalInfo !=
nullptr)
21376 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21380 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21381 "RISC-V target doesn't support this non-temporal domain.");
21383 NontemporalLevel -= 2;
21385 if (NontemporalLevel & 0b1)
21387 if (NontemporalLevel & 0b10)
21400 return TargetFlags;
21410 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21413 return Subtarget.hasStdExtZbb() &&
21426 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21427 Op == Instruction::And ||
Op == Instruction::Or ||
21428 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21429 Op == Instruction::ShuffleVector ||
Op == Instruction::Load)
21437 !isa<ReturnInst>(&Inst))
21440 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21441 if (AI->getAllocatedType()->isScalableTy())
21449RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21457 if (!Subtarget.hasShortForwardBranchOpt())
21459 EVT VT =
N->getValueType(0);
21460 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21464 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21469bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21470 EVT VT,
const APInt &AndMask)
const {
21471 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21472 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21476unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21481template <
typename Arg>
21482void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21485 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21487 auto It = ArgList.
begin();
21488 MVT FirstArgRegType = It->VT;
21491 if (It == ArgList.
end() || It->Flags.isSplit())
21503 for (; It != ArgList.
end(); ++It)
21504 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21510 if (isHomogeneousScalableVectorType(ArgList)) {
21512 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21515 bool FirstVMaskAssigned =
false;
21516 for (
const auto &OutArg : ArgList) {
21517 MVT RegisterVT = OutArg.VT;
21527 RVVArgInfos.push_back({1, RegisterVT,
true});
21528 FirstVMaskAssigned =
true;
21532 RVVArgInfos.push_back({1, RegisterVT,
false});
21539void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21544 bool FirstVMaskAssigned =
false;
21545 for (
Type *Ty : TypeList) {
21555 RVVArgInfos.push_back(
21561 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21576 if (!FirstVMaskAssigned &&
21578 RVVArgInfos.push_back({1, RegisterVT,
true});
21579 FirstVMaskAssigned =
true;
21583 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21589void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21590 unsigned StartReg) {
21591 assert((StartReg % LMul) == 0 &&
21592 "Start register number should be multiple of lmul");
21611 for (
unsigned i = 0; i < NF; ++i)
21613 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21615 AllocatedPhysRegs.push_back(
MCPhysReg());
21621void RVVArgDispatcher::compute() {
21623 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21626 AllocatedPhysRegs.push_back(RISCV::V0);
21632 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21633 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21634 StartReg += RegsNeeded) {
21635 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21636 if ((AssignedMap & Map) == 0) {
21637 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21638 AssignedMap |=
Map;
21643 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21646 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21647 allocate(RVVArgInfos[i]);
21651 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21652 return AllocatedPhysRegs[CurIdx++];
21657#define GET_RISCVVIntrinsicsTable_IMPL
21658#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)