26#include "llvm/IR/IntrinsicsAMDGPU.h"
33#define DEBUG_TYPE "AMDGPUtti"
36 "amdgpu-unroll-threshold-private",
37 cl::desc(
"Unroll threshold for AMDGPU if private memory used in a loop"),
41 "amdgpu-unroll-threshold-local",
42 cl::desc(
"Unroll threshold for AMDGPU if local memory used in a loop"),
46 "amdgpu-unroll-threshold-if",
47 cl::desc(
"Unroll threshold increment for AMDGPU for each if statement inside loop"),
51 "amdgpu-unroll-runtime-local",
52 cl::desc(
"Allow runtime unroll for AMDGPU if local memory used in a loop"),
56 "amdgpu-unroll-max-block-to-analyze",
57 cl::desc(
"Inner loop block size threshold to analyze in unroll for AMDGPU"),
62 cl::desc(
"Cost of alloca argument"));
70 cl::desc(
"Maximum alloca size to use for inline cost"));
75 cl::desc(
"Maximum number of BBs allowed in a function after inlining"
76 " (compile time constraint)"));
84 for (
const Value *V :
I->operand_values()) {
87 if (
const PHINode *
PHI = dyn_cast<PHINode>(V)) {
89 return SubLoop->contains(PHI); }))
99 TargetTriple(
TM->getTargetTriple()),
101 TLI(ST->getTargetLowering()) {}
106 const Function &
F = *L->getHeader()->getParent();
108 F.getFnAttributeAsParsedInteger(
"amdgpu-unroll-threshold", 300);
109 UP.
MaxCount = std::numeric_limits<unsigned>::max();
122 const unsigned MaxAlloca = (256 - 16) * 4;
128 if (
MDNode *LoopUnrollThreshold =
130 if (LoopUnrollThreshold->getNumOperands() == 2) {
131 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
132 LoopUnrollThreshold->getOperand(1));
133 if (MetaThresholdValue) {
139 ThresholdPrivate = std::min(ThresholdPrivate, UP.
Threshold);
140 ThresholdLocal = std::min(ThresholdLocal, UP.
Threshold);
145 unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);
147 const DataLayout &
DL = BB->getModule()->getDataLayout();
148 unsigned LocalGEPsSeen = 0;
151 return SubLoop->contains(BB); }))
160 if (
const BranchInst *Br = dyn_cast<BranchInst>(&
I)) {
161 if (UP.
Threshold < MaxBoost && Br->isConditional()) {
164 if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) ||
165 (L->contains(Succ1) && L->isLoopExiting(Succ1)))
171 << *L <<
" due to " << *Br <<
'\n');
183 unsigned AS =
GEP->getAddressSpace();
184 unsigned Threshold = 0;
186 Threshold = ThresholdPrivate;
188 Threshold = ThresholdLocal;
203 if (AllocaSize > MaxAlloca)
212 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
213 (!isa<GlobalVariable>(
GEP->getPointerOperand()) &&
214 !isa<Argument>(
GEP->getPointerOperand())))
217 << *L <<
" due to LDS use.\n");
222 bool HasLoopDef =
false;
225 if (!Inst || L->isLoopInvariant(
Op))
229 return SubLoop->contains(Inst); }))
253 << *L <<
" due to " << *
GEP <<
'\n');
276 AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableSIScheduler,
277 AMDGPU::FeatureEnableUnsafeDSOffsetFolding, AMDGPU::FeatureFlatForGlobal,
278 AMDGPU::FeaturePromoteAlloca, AMDGPU::FeatureUnalignedScratchAccess,
279 AMDGPU::FeatureUnalignedAccessMode,
281 AMDGPU::FeatureAutoWaitcntBeforeBarrier,
284 AMDGPU::FeatureSGPRInitBug, AMDGPU::FeatureXNACK,
285 AMDGPU::FeatureTrapHandler,
289 AMDGPU::FeatureSRAMECC,
292 AMDGPU::FeatureFastFMAF32, AMDGPU::HalfRate64Ops};
297 TLI(ST->getTargetLowering()), CommonTTI(
TM,
F),
298 IsGraphics(AMDGPU::isGraphics(
F.getCallingConv())) {
301 HasFP64FP16Denormals =
338 if (Opcode == Instruction::Load || Opcode == Instruction::Store)
339 return 32 * 4 / ElemWidth;
346 unsigned ChainSizeInBytes,
348 unsigned VecRegBitWidth = VF * LoadSize;
351 return 128 / LoadSize;
357 unsigned ChainSizeInBytes,
359 unsigned VecRegBitWidth = VF * StoreSize;
360 if (VecRegBitWidth > 128)
361 return 128 / StoreSize;
385 unsigned AddrSpace)
const {
398 unsigned AddrSpace)
const {
404 unsigned AddrSpace)
const {
421 unsigned DestAddrSpace,
unsigned SrcAlign,
unsigned DestAlign,
422 std::optional<uint32_t> AtomicElementSize)
const {
424 if (AtomicElementSize)
427 unsigned MinAlign = std::min(SrcAlign, DestAlign);
452 unsigned RemainingBytes,
unsigned SrcAddrSpace,
unsigned DestAddrSpace,
453 unsigned SrcAlign,
unsigned DestAlign,
454 std::optional<uint32_t> AtomicCpySize)
const {
455 assert(RemainingBytes < 16);
459 OpsOut,
Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
460 DestAlign, AtomicCpySize);
462 unsigned MinAlign = std::min(SrcAlign, DestAlign);
466 while (RemainingBytes >= 8) {
472 while (RemainingBytes >= 4) {
479 while (RemainingBytes >= 2) {
485 while (RemainingBytes) {
503 case Intrinsic::amdgcn_ds_ordered_add:
504 case Intrinsic::amdgcn_ds_ordered_swap:
505 case Intrinsic::amdgcn_ds_fadd:
506 case Intrinsic::amdgcn_ds_fmin:
507 case Intrinsic::amdgcn_ds_fmax: {
508 auto *Ordering = dyn_cast<ConstantInt>(Inst->
getArgOperand(2));
509 auto *Volatile = dyn_cast<ConstantInt>(Inst->
getArgOperand(4));
510 if (!Ordering || !Volatile)
513 unsigned OrderingVal = Ordering->getZExtValue();
520 Info.WriteMem =
true;
521 Info.IsVolatile = !Volatile->isZero();
536 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
541 unsigned NElts = LT.second.isVector() ?
542 LT.second.getVectorNumElements() : 1;
551 return get64BitInstrCost(
CostKind) * LT.first * NElts;
554 NElts = (NElts + 1) / 2;
557 return getFullRateInstrCost() * LT.first * NElts;
563 if (SLT == MVT::i64) {
565 return 2 * getFullRateInstrCost() * LT.first * NElts;
569 NElts = (NElts + 1) / 2;
571 return LT.first * NElts * getFullRateInstrCost();
573 const int QuarterRateCost = getQuarterRateInstrCost(
CostKind);
574 if (SLT == MVT::i64) {
575 const int FullRateCost = getFullRateInstrCost();
576 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
580 NElts = (NElts + 1) / 2;
583 return QuarterRateCost * NElts * LT.first;
590 if (
const auto *
FAdd = dyn_cast<BinaryOperator>(*CxtI->
user_begin())) {
595 if (ST->
has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)
610 NElts = (NElts + 1) / 2;
612 return LT.first * NElts * get64BitInstrCost(
CostKind);
615 NElts = (NElts + 1) / 2;
617 if (SLT == MVT::f32 || SLT == MVT::f16)
618 return LT.first * NElts * getFullRateInstrCost();
624 if (SLT == MVT::f64) {
630 Cost += 3 * getFullRateInstrCost();
632 return LT.first *
Cost * NElts;
637 if ((SLT == MVT::f32 && !HasFP32Denormals) ||
639 return LT.first * getQuarterRateInstrCost(
CostKind) * NElts;
650 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost(
CostKind);
651 return LT.first *
Cost * NElts;
659 int Cost = getQuarterRateInstrCost(
CostKind) + getFullRateInstrCost();
660 return LT.first *
Cost * NElts;
663 if (SLT == MVT::f32 || SLT == MVT::f16) {
665 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
666 1 * getQuarterRateInstrCost(
CostKind);
668 if (!HasFP32Denormals) {
670 Cost += 2 * getFullRateInstrCost();
673 return LT.first * NElts *
Cost;
694 case Intrinsic::round:
695 case Intrinsic::uadd_sat:
696 case Intrinsic::usub_sat:
697 case Intrinsic::sadd_sat:
698 case Intrinsic::ssub_sat:
708 if (ICA.
getID() == Intrinsic::fabs)
717 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(
RetTy);
719 unsigned NElts = LT.second.isVector() ?
720 LT.second.getVectorNumElements() : 1;
725 return LT.first * NElts * get64BitInstrCost(
CostKind);
729 NElts = (NElts + 1) / 2;
732 unsigned InstRate = getQuarterRateInstrCost(
CostKind);
734 switch (ICA.
getID()) {
737 : getQuarterRateInstrCost(
CostKind);
739 case Intrinsic::uadd_sat:
740 case Intrinsic::usub_sat:
741 case Intrinsic::sadd_sat:
742 case Intrinsic::ssub_sat:
743 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
744 if (
any_of(ValidSatTys, [<](
MVT M) {
return M == LT.second; }))
749 return LT.first * NElts * InstRate;
755 assert((
I ==
nullptr ||
I->getOpcode() == Opcode) &&
756 "Opcode should reflect passed instruction.");
759 const int CBrCost = SCost ? 5 : 7;
761 case Instruction::Br: {
763 auto BI = dyn_cast_or_null<BranchInst>(
I);
764 if (BI && BI->isUnconditional())
765 return SCost ? 1 : 4;
770 case Instruction::Switch: {
771 auto SI = dyn_cast_or_null<SwitchInst>(
I);
774 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
776 case Instruction::Ret:
777 return SCost ? 1 : 10;
784 std::optional<FastMathFlags> FMF,
796 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
797 return LT.first * getFullRateInstrCost();
811 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
812 return LT.first * getHalfRateInstrCost(
CostKind);
820 case Instruction::ExtractElement:
821 case Instruction::InsertElement: {
836 return Index == ~0u ? 2 : 0;
850 if (Indices.
size() > 1)
858 const int TargetOutputIdx = Indices.
empty() ? -1 : Indices[0];
861 for (
auto &TC : TargetConstraints) {
866 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
872 TRI, TC.ConstraintCode, TC.ConstraintVT).second;
876 if (!RC || !
TRI->isSGPRClass(RC))
886 cast<MetadataAsValue>(ReadReg->
getArgOperand(0))->getMetadata();
888 cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();
907 if (
const Argument *
A = dyn_cast<Argument>(V))
916 if (
const LoadInst *Load = dyn_cast<LoadInst>(V))
924 if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
927 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
928 if (Intrinsic->getIntrinsicID() == Intrinsic::read_register)
935 if (
const CallInst *CI = dyn_cast<CallInst>(V)) {
936 if (CI->isInlineAsm())
942 if (isa<InvokeInst>(V))
949 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
952 if (
const CallInst *CI = dyn_cast<CallInst>(V)) {
953 if (CI->isInlineAsm())
971 if (
match(V,
m_LShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
973 match(V,
m_AShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
975 const Function *
F = cast<Instruction>(V)->getFunction();
981 if (
match(V,
m_c_And(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
983 const Function *
F = cast<Instruction>(V)->getFunction();
998 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) {
999 switch (Intrinsic->getIntrinsicID()) {
1002 case Intrinsic::amdgcn_if:
1003 case Intrinsic::amdgcn_else: {
1005 return Indices.
size() == 1 && Indices[0] == 1;
1022 case Intrinsic::amdgcn_ds_fadd:
1023 case Intrinsic::amdgcn_ds_fmin:
1024 case Intrinsic::amdgcn_ds_fmax:
1025 case Intrinsic::amdgcn_is_shared:
1026 case Intrinsic::amdgcn_is_private:
1027 case Intrinsic::amdgcn_flat_atomic_fadd:
1028 case Intrinsic::amdgcn_flat_atomic_fmax:
1029 case Intrinsic::amdgcn_flat_atomic_fmin:
1030 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1031 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1041 Value *NewV)
const {
1044 case Intrinsic::amdgcn_ds_fadd:
1045 case Intrinsic::amdgcn_ds_fmin:
1046 case Intrinsic::amdgcn_ds_fmax: {
1048 if (!IsVolatile->isZero())
1059 case Intrinsic::amdgcn_is_shared:
1060 case Intrinsic::amdgcn_is_private: {
1061 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
1069 case Intrinsic::ptrmask: {
1075 bool DoTruncate =
false;
1079 if (!
TM.isNoopAddrSpaceCast(OldAS, NewAS)) {
1097 MaskTy =
B.getInt32Ty();
1098 MaskOp =
B.CreateTrunc(MaskOp, MaskTy);
1101 return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->
getType(), MaskTy},
1104 case Intrinsic::amdgcn_flat_atomic_fadd:
1105 case Intrinsic::amdgcn_flat_atomic_fmax:
1106 case Intrinsic::amdgcn_flat_atomic_fmin:
1107 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1108 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
1116 {DestTy, SrcTy, DestTy});
1135 if (IsExtractSubvector)
1139 if (cast<FixedVectorType>(VT)->getNumElements() == 2 &&
1155 if (IsExtractSubvector)
1165 =
static_cast<const GCNSubtarget *
>(
TM.getSubtargetImpl(*Caller));
1167 =
static_cast<const GCNSubtarget *
>(
TM.getSubtargetImpl(*Callee));
1169 const FeatureBitset &CallerBits = CallerST->getFeatureBits();
1170 const FeatureBitset &CalleeBits = CalleeST->getFeatureBits();
1172 FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList;
1173 FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList;
1174 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
1184 if (Callee->hasFnAttribute(Attribute::AlwaysInline) ||
1185 Callee->hasFnAttribute(Attribute::InlineHint))
1191 if (Callee->size() == 1)
1193 size_t BBSize = Caller->size() + Callee->size() - 1;
1203 const int NrOfSGPRUntilSpill = 26;
1204 const int NrOfVGPRUntilSpill = 32;
1208 unsigned adjustThreshold = 0;
1214 for (
auto ArgVT : ValueVTs) {
1218 SGPRsInUse += CCRegNum;
1220 VGPRsInUse += CCRegNum;
1230 ArgStackCost +=
const_cast<GCNTTIImpl *
>(TTIImpl)->getMemoryOpCost(
1233 ArgStackCost +=
const_cast<GCNTTIImpl *
>(TTIImpl)->getMemoryOpCost(
1239 adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *
1241 adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *
1243 return adjustThreshold;
1252 unsigned AllocaSize = 0;
1255 PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());
1259 unsigned AddrSpace = Ty->getAddressSpace();
1307 static_assert(InlinerVectorBonusPercent == 0,
"vector bonus assumed to be 0");
1311 return BB.getTerminator()->getNumSuccessors() > 1;
1314 Threshold += Threshold / 2;
1320 unsigned AllocaThresholdBonus = (Threshold * ArgAllocaSize) / AllocaSize;
1322 return AllocaThresholdBonus;
1338 ? getFullRateInstrCost()
1339 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(
CostKind)
1340 : getQuarterRateInstrCost(
CostKind);
1343std::pair<InstructionCost, MVT>
1344GCNTTIImpl::getTypeLegalizationCost(
Type *Ty)
const {
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static const Function * getParent(const Value *V)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool hasMadMacF32Insts() const
unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const
Return the maximum workitem ID value in the function, for the given (0, 1, 2) dimension.
unsigned getWavefrontSizeLog2() const
bool has16BitInsts() const
bool hasFastFMAF32() const
bool isSingleLaneExecution(const Function &Kernel) const
Return true if only a single workitem can be active in a wave.
bool hasVOP3PInsts() const
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE)
int64_t getMaxMemIntrinsicInlineSizeThreshold() const
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
an instruction to allocate memory on the stack
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind)
Get intrinsic cost based on arguments.
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Op0, Value *Op1)
TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind, ArrayRef< int > Mask, VectorType *Ty, int &Index, VectorType *&SubTy) const
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind)
Try to calculate op costs for min/max reduction operations.
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef< const Value * > Args=std::nullopt, const Instruction *CxtI=nullptr)
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr)
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args=std::nullopt, const Instruction *CxtI=nullptr)
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind)
std::pair< InstructionCost, MVT > getTypeLegalizationCost(Type *Ty) const
Estimate the cost of type-legalization and the legalized type.
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getFalse(LLVMContext &Context)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
Container class for subtarget features.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool hasUsableDivScaleConditionOutput() const
Condition output from div_scale is usable.
const SIRegisterInfo * getRegisterInfo() const override
bool hasUnalignedScratchAccess() const
bool hasPackedFP32Ops() const
bool hasFullRate64Ops() const
unsigned getMaxPrivateElementSize(bool ForBufferRSrc=false) const
TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind)
InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index, Value *Op0, Value *Op1)
bool isAlwaysUniform(const Value *V) const
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
int64_t getMaxMemIntrinsicInlineSizeThreshold() const
bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind)
bool isInlineAsmSourceOfDivergence(const CallInst *CI, ArrayRef< unsigned > Indices={}) const
Analyze if the results of inline asm are divergent.
bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef< const Value * > Args=std::nullopt, const Instruction *CxtI=nullptr)
unsigned getNumberOfRegisters(unsigned RCID) const
bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
bool shouldPrefetchAddressSpace(unsigned AS) const override
unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
unsigned getMaxInterleaveFactor(ElementCount VF)
unsigned getInliningThresholdMultiplier() const
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE)
unsigned getMinVectorRegisterBitWidth() const
bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const
unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
unsigned getPrefetchDistance() const override
How much before a load we should place the prefetch instruction.
Value * rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV, Value *NewV) const
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const
void getMemcpyLoopResidualLoweringType(SmallVectorImpl< Type * > &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, std::optional< uint32_t > AtomicCpySize) const
bool collectFlatAddressOperands(SmallVectorImpl< int > &OpIndexes, Intrinsic::ID IID) const
unsigned adjustInliningThreshold(const CallBase *CB) const
bool areInlineCompatible(const Function *Caller, const Function *Callee) const
unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr)
unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const
bool isSourceOfDivergence(const Value *V) const
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args=std::nullopt, const Instruction *CxtI=nullptr)
bool hasBranchDivergence(const Function *F=nullptr) const
Type * getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, std::optional< uint32_t > AtomicElementSize) const
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Module * getParent()
Get the module that this global value is contained inside of...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< CostType > getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
bool hasApproxFunc() const LLVM_READONLY
Determine whether the approximate-math-functions flag is set.
bool hasAllowContract() const LLVM_READONLY
Determine whether the allow-contract flag is set.
Type * getReturnType() const
Intrinsic::ID getID() const
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Represents a single loop in the control flow graph.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
The main scalar evolution driver.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
const TargetMachine & getTargetMachine() const
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
Primary interface to the complete machine description for the target machine.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
Type * getElementType() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
bool isFlatGlobalAddrSpace(unsigned AS)
bool isArgPassedInSGPR(const Argument *A)
bool isIntrinsicAlwaysUniform(unsigned IntrID)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
bool isExtendedGlobalAddrSpace(unsigned AS)
@ C
The default llvm calling convention, compatible with C.
@ ADD
Simple integer binary arithmetic operators.
@ FADD
Simple binary floating point operators.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SHL
Shift and rotation operations.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
bool match(Val *V, const Pattern &P)
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
specific_fpval m_FPOne()
Match a float 1.0 or vector with all elements equal to 1.0.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
MDNode * findOptionMDForLoop(const Loop *TheLoop, StringRef Name)
Find string metadata for a loop.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
AtomicOrdering
Atomic ordering for LLVM's memory model.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
This struct is a compact representation of a valid (non-zero power of two) alignment.
static constexpr DenormalMode getPreserveSign()
uint64_t getScalarSizeInBits() const
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Information about a load/store intrinsic defined by the target.
bool isInlineCompatible(SIModeRegisterDefaults CalleeMode) const