14#ifndef LLVM_ANALYSIS_TARGETTRANSFORMINFOIMPL_H
15#define LLVM_ANALYSIS_TARGETTRANSFORMINFOIMPL_H
56 if (!isa<Constant>(Operand))
69 return SI.getNumCases();
132 std::pair<const Value *, unsigned>
134 return std::make_pair(
nullptr, -1);
143 assert(
F &&
"A concrete function must be provided to this routine.");
150 if (
F->isIntrinsic())
153 if (
F->hasLocalLinkage() || !
F->hasName())
159 if (
Name ==
"copysign" ||
Name ==
"copysignf" ||
Name ==
"copysignl" ||
169 Name ==
"exp2l" ||
Name ==
"exp2f" ||
Name ==
"floor" ||
170 Name ==
"floorf" ||
Name ==
"ceil" ||
Name ==
"round" ||
196 std::optional<Value *>
199 bool &KnownBitsComputed)
const {
207 SimplifyAndSetOp)
const {
225 bool HasBaseReg, int64_t Scale,
unsigned AddrSpace,
227 int64_t ScalableOffset = 0)
const {
230 return !BaseGV && BaseOffset == 0 && (Scale == 0 || Scale == 1);
298 Align Alignment)
const {
332 unsigned AddrSpace)
const {
335 Scale, AddrSpace,
nullptr,
347 bool useAA()
const {
return false; }
362 const APInt &DemandedElts,
363 bool Insert,
bool Extract,
384 bool IsZeroCmp)
const {
394 return isa<SelectInst>(
I) &&
407 unsigned *
Fast)
const {
462 return "Generic::Unknown Register Class";
464 return "Generic::ScalarRC";
466 return "Generic::VectorRC";
489 unsigned getMaximumVF(
unsigned ElemWidth,
unsigned Opcode)
const {
return 0; }
493 const Instruction &
I,
bool &AllowPromotionWithoutCommonHeader)
const {
494 AllowPromotionWithoutCommonHeader =
false;
499 std::optional<unsigned>
510 std::optional<unsigned>
526 unsigned NumStridedMemAccesses,
527 unsigned NumPrefetches,
bool HasCall)
const {
543 auto IsWidenableCondition = [](
const Value *V) {
544 if (
auto *II = dyn_cast<IntrinsicInst>(V))
545 if (II->
getIntrinsicID() == Intrinsic::experimental_widenable_condition)
554 case Instruction::FDiv:
555 case Instruction::FRem:
556 case Instruction::SDiv:
557 case Instruction::SRem:
558 case Instruction::UDiv:
559 case Instruction::URem:
562 case Instruction::And:
563 case Instruction::Or:
564 if (
any_of(Args, IsWidenableCondition))
600 case Instruction::IntToPtr: {
601 unsigned SrcSize = Src->getScalarSizeInBits();
607 case Instruction::PtrToInt: {
608 unsigned DstSize = Dst->getScalarSizeInBits();
614 case Instruction::BitCast:
615 if (Dst == Src || (Dst->isPointerTy() && Src->isPointerTy()))
619 case Instruction::Trunc: {
633 unsigned Index)
const {
662 unsigned Index)
const {
667 const APInt &DemandedDstElts,
694 const Value *
Ptr,
bool VariableMask,
702 const Value *
Ptr,
bool VariableMask,
712 bool UseMaskForCond,
bool UseMaskForGaps)
const {
718 switch (ICA.
getID()) {
721 case Intrinsic::allow_runtime_check:
722 case Intrinsic::allow_ubsan_check:
723 case Intrinsic::annotation:
724 case Intrinsic::assume:
725 case Intrinsic::sideeffect:
726 case Intrinsic::pseudoprobe:
727 case Intrinsic::arithmetic_fence:
728 case Intrinsic::dbg_assign:
729 case Intrinsic::dbg_declare:
730 case Intrinsic::dbg_value:
731 case Intrinsic::dbg_label:
732 case Intrinsic::invariant_start:
733 case Intrinsic::invariant_end:
734 case Intrinsic::launder_invariant_group:
735 case Intrinsic::strip_invariant_group:
736 case Intrinsic::is_constant:
737 case Intrinsic::lifetime_start:
738 case Intrinsic::lifetime_end:
739 case Intrinsic::experimental_noalias_scope_decl:
740 case Intrinsic::objectsize:
741 case Intrinsic::ptr_annotation:
742 case Intrinsic::var_annotation:
743 case Intrinsic::experimental_gc_result:
744 case Intrinsic::experimental_gc_relocate:
745 case Intrinsic::coro_alloc:
746 case Intrinsic::coro_begin:
747 case Intrinsic::coro_free:
748 case Intrinsic::coro_end:
749 case Intrinsic::coro_frame:
750 case Intrinsic::coro_size:
751 case Intrinsic::coro_align:
752 case Intrinsic::coro_suspend:
753 case Intrinsic::coro_subfn_addr:
754 case Intrinsic::threadlocal_address:
755 case Intrinsic::experimental_widenable_condition:
756 case Intrinsic::ssa_copy:
773 const SCEV *)
const {
778 std::optional<FastMathFlags> FMF,
820 Type *ExpectedType)
const {
826 unsigned SrcAddrSpace,
unsigned DestAddrSpace,
827 unsigned SrcAlign,
unsigned DestAlign,
828 std::optional<uint32_t> AtomicElementSize)
const {
835 unsigned RemainingBytes,
unsigned SrcAddrSpace,
unsigned DestAddrSpace,
836 unsigned SrcAlign,
unsigned DestAlign,
837 std::optional<uint32_t> AtomicCpySize)
const {
838 unsigned OpSizeInBytes = AtomicCpySize ? *AtomicCpySize : 1;
840 for (
unsigned i = 0; i != RemainingBytes; i += OpSizeInBytes)
846 return (Caller->getFnAttribute(
"target-cpu") ==
847 Callee->getFnAttribute(
"target-cpu")) &&
848 (Caller->getFnAttribute(
"target-features") ==
849 Callee->getFnAttribute(
"target-features"));
853 unsigned DefaultCallPenalty)
const {
854 return DefaultCallPenalty;
859 return (Caller->getFnAttribute(
"target-cpu") ==
860 Callee->getFnAttribute(
"target-cpu")) &&
861 (Caller->getFnAttribute(
"target-features") ==
862 Callee->getFnAttribute(
"target-features"));
882 unsigned AddrSpace)
const {
887 unsigned AddrSpace)
const {
899 unsigned ChainSizeInBytes,
905 unsigned ChainSizeInBytes,
935 Align Alignment)
const {
954 if (isa<ConstantDataVector>(Val) || isa<ConstantVector>(Val)) {
955 const auto *VectorValue = cast<Constant>(Val);
959 auto *VT = cast<FixedVectorType>(Val->
getType());
965 unsigned MaxRequiredSize =
966 VT->getElementType()->getPrimitiveSizeInBits().getFixedValue();
968 unsigned MinRequiredSize = 0;
969 for (
unsigned i = 0, e = VT->getNumElements(); i < e; ++i) {
970 if (
auto *IntElement =
971 dyn_cast<ConstantInt>(VectorValue->getAggregateElement(i))) {
972 bool signedElement = IntElement->getValue().isNegative();
974 unsigned ElementMinRequiredSize =
975 IntElement->getValue().getSignificantBits() - 1;
979 MinRequiredSize = std::max(MinRequiredSize, ElementMinRequiredSize);
982 return MaxRequiredSize;
985 return MinRequiredSize;
988 if (
const auto *CI = dyn_cast<ConstantInt>(Val)) {
989 isSigned = CI->getValue().isNegative();
990 return CI->getValue().getSignificantBits() - 1;
993 if (
const auto *Cast = dyn_cast<SExtInst>(Val)) {
995 return Cast->getSrcTy()->getScalarSizeInBits() - 1;
998 if (
const auto *Cast = dyn_cast<ZExtInst>(Val)) {
1000 return Cast->getSrcTy()->getScalarSizeInBits();
1008 return Ptr && isa<SCEVAddRecExpr>(
Ptr);
1020 int64_t MergeDistance)
const {
1034template <
typename T>
1048 assert(PointeeType &&
Ptr &&
"can't get GEPCost of nullptr");
1049 auto *BaseGV = dyn_cast<GlobalValue>(
Ptr->stripPointerCasts());
1050 bool HasBaseReg = (BaseGV ==
nullptr);
1053 APInt BaseOffset(PtrSizeBits, 0);
1057 Type *TargetType =
nullptr;
1065 TargetType = GTI.getIndexedType();
1068 const ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*
I);
1071 ConstIdx = dyn_cast<ConstantInt>(
Splat);
1072 if (
StructType *STy = GTI.getStructTypeOrNull()) {
1074 assert(ConstIdx &&
"Unexpected GEP index");
1082 int64_t ElementSize =
1083 GTI.getSequentialElementStride(
DL).getFixedValue();
1092 Scale = ElementSize;
1107 AccessType = TargetType;
1114 Ptr->getType()->getPointerAddressSpace()))
1139 for (
const Value *V : Ptrs) {
1140 const auto *
GEP = dyn_cast<GetElementPtrInst>(V);
1143 if (
Info.isSameBase() && V !=
Base) {
1144 if (
GEP->hasAllConstantIndices())
1153 GEP->getPointerOperand(),
1165 auto *TargetTTI =
static_cast<T *
>(
this);
1168 auto *CB = dyn_cast<CallBase>(U);
1169 if (CB && !isa<IntrinsicInst>(U)) {
1170 if (
const Function *
F = CB->getCalledFunction()) {
1171 if (!TargetTTI->isLoweredToCall(
F))
1180 Type *Ty = U->getType();
1182 auto *
I = dyn_cast<Instruction>(U);
1186 case Instruction::Call: {
1187 assert(isa<IntrinsicInst>(U) &&
"Unexpected non-intrinsic call");
1188 auto *Intrinsic = cast<IntrinsicInst>(U);
1190 return TargetTTI->getIntrinsicInstrCost(CostAttrs,
CostKind);
1192 case Instruction::Br:
1193 case Instruction::Ret:
1194 case Instruction::PHI:
1195 case Instruction::Switch:
1196 return TargetTTI->getCFInstrCost(Opcode,
CostKind,
I);
1197 case Instruction::ExtractValue:
1198 case Instruction::Freeze:
1200 case Instruction::Alloca:
1201 if (cast<AllocaInst>(U)->isStaticAlloca())
1204 case Instruction::GetElementPtr: {
1205 const auto *
GEP = cast<GEPOperator>(U);
1206 Type *AccessType =
nullptr;
1209 if (
GEP->hasOneUser() &&
I)
1210 AccessType =
I->user_back()->getAccessType();
1212 return TargetTTI->getGEPCost(
GEP->getSourceElementType(),
1216 case Instruction::Add:
1217 case Instruction::FAdd:
1218 case Instruction::Sub:
1219 case Instruction::FSub:
1220 case Instruction::Mul:
1221 case Instruction::FMul:
1222 case Instruction::UDiv:
1223 case Instruction::SDiv:
1224 case Instruction::FDiv:
1225 case Instruction::URem:
1226 case Instruction::SRem:
1227 case Instruction::FRem:
1228 case Instruction::Shl:
1229 case Instruction::LShr:
1230 case Instruction::AShr:
1231 case Instruction::And:
1232 case Instruction::Or:
1233 case Instruction::Xor:
1234 case Instruction::FNeg: {
1237 if (Opcode != Instruction::FNeg)
1239 return TargetTTI->getArithmeticInstrCost(Opcode, Ty,
CostKind, Op1Info,
1242 case Instruction::IntToPtr:
1243 case Instruction::PtrToInt:
1244 case Instruction::SIToFP:
1245 case Instruction::UIToFP:
1246 case Instruction::FPToUI:
1247 case Instruction::FPToSI:
1248 case Instruction::Trunc:
1249 case Instruction::FPTrunc:
1250 case Instruction::BitCast:
1251 case Instruction::FPExt:
1252 case Instruction::SExt:
1253 case Instruction::ZExt:
1254 case Instruction::AddrSpaceCast: {
1256 return TargetTTI->getCastInstrCost(
1259 case Instruction::Store: {
1260 auto *SI = cast<StoreInst>(U);
1263 return TargetTTI->getMemoryOpCost(Opcode, ValTy, SI->getAlign(),
1264 SI->getPointerAddressSpace(),
CostKind,
1267 case Instruction::Load: {
1271 auto *LI = cast<LoadInst>(U);
1272 Type *LoadType = U->getType();
1282 if (
const TruncInst *TI = dyn_cast<TruncInst>(*LI->user_begin()))
1283 LoadType = TI->getDestTy();
1285 return TargetTTI->getMemoryOpCost(Opcode, LoadType, LI->getAlign(),
1287 {TTI::OK_AnyValue, TTI::OP_None},
I);
1289 case Instruction::Select: {
1290 const Value *Op0, *Op1;
1301 return TargetTTI->getArithmeticInstrCost(
1306 return TargetTTI->getCmpSelInstrCost(Opcode, U->getType(), CondTy,
1310 case Instruction::ICmp:
1311 case Instruction::FCmp: {
1314 return TargetTTI->getCmpSelInstrCost(Opcode, ValTy, U->getType(),
1315 I ? cast<CmpInst>(
I)->getPredicate()
1319 case Instruction::InsertElement: {
1320 auto *IE = dyn_cast<InsertElementInst>(U);
1324 if (
auto *CI = dyn_cast<ConstantInt>(
Operands[2]))
1325 if (CI->getValue().getActiveBits() <= 32)
1326 Idx = CI->getZExtValue();
1327 return TargetTTI->getVectorInstrCost(*IE, Ty,
CostKind,
Idx);
1329 case Instruction::ShuffleVector: {
1330 auto *Shuffle = dyn_cast<ShuffleVectorInst>(U);
1334 auto *VecTy = cast<VectorType>(U->getType());
1337 int NumSubElts, SubIndex;
1340 if (Shuffle->changesLength()) {
1342 if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding())
1345 if (Shuffle->isExtractSubvectorMask(SubIndex))
1350 if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex))
1351 return TargetTTI->getShuffleCost(
1356 int ReplicationFactor, VF;
1357 if (Shuffle->isReplicationMask(ReplicationFactor, VF)) {
1361 DemandedDstElts.
setBit(
I.index());
1363 return TargetTTI->getReplicationShuffleCost(
1364 VecSrcTy->getElementType(), ReplicationFactor, VF,
1368 bool IsUnary = isa<UndefValue>(
Operands[1]);
1369 NumSubElts = VecSrcTy->getElementCount().getKnownMinValue();
1375 if (Shuffle->increasesLength()) {
1376 for (
int &M : AdjustMask)
1377 M = M >= NumSubElts ? (M + (Mask.size() - NumSubElts)) : M;
1379 return TargetTTI->getShuffleCost(
1393 std::iota(ExtractMask.
begin(), ExtractMask.
end(), 0);
1394 return ShuffleCost + TargetTTI->getShuffleCost(
1396 ExtractMask,
CostKind, 0, VecTy, {}, Shuffle);
1399 if (Shuffle->isIdentity())
1402 if (Shuffle->isReverse())
1406 if (Shuffle->isSelect())
1410 if (Shuffle->isTranspose())
1415 if (Shuffle->isZeroEltSplat())
1420 if (Shuffle->isSingleSource())
1425 if (Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex))
1426 return TargetTTI->getShuffleCost(
1431 if (Shuffle->isSplice(SubIndex))
1433 SubIndex,
nullptr,
Operands, Shuffle);
1438 case Instruction::ExtractElement: {
1439 auto *EEI = dyn_cast<ExtractElementInst>(U);
1443 if (
auto *CI = dyn_cast<ConstantInt>(
Operands[1]))
1444 if (CI->getValue().getActiveBits() <= 32)
1445 Idx = CI->getZExtValue();
1447 return TargetTTI->getVectorInstrCost(*EEI, DstTy,
CostKind,
Idx);
1457 auto *TargetTTI =
static_cast<T *
>(
this);
Analysis containing CSE Info
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
mir Rename Register Operands
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
unsigned getBitWidth() const
Return the number of bits in the APInt.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
int64_t getSExtValue() const
Get sign extended value.
an instruction to allocate memory on the stack
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A cache of @llvm.assume calls within a function.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
bool isLegalInteger(uint64_t Width) const
Returns true if the specified type is known to be a native integer type supported by the CPU.
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Convenience struct for specifying and reasoning about fast-math flags.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
The core instruction combiner logic.
static InstructionCost getInvalid(CostType Val=0)
Intrinsic::ID getID() const
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Represents a single loop in the control flow graph.
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This node represents a polynomial recurrence on the trip count of the specified loop.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
This class represents a constant integer value.
const APInt & getAPInt() const
This class represents an analyzed expression in the program.
The main scalar evolution driver.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
TypeSize getElementOffset(unsigned Idx) const
Class to represent struct types.
Provides information about what library functions are available for the current target.
This class represents a truncation of integer types.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
This is the common base class for vector predication intrinsics.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
This is an optimization pass for GlobalISel generic memory operations.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
constexpr int PoisonMaskElem
constexpr unsigned BitWidth
gep_type_iterator gep_type_begin(const User *GEP)
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Attributes of a target dependent hardware loop.
Information about a load/store intrinsic defined by the target.