51#include "llvm/IR/IntrinsicsAMDGPU.h"
105static SGPRRegisterRegAlloc
106defaultSGPRRegAlloc(
"default",
107 "pick SGPR register allocator based on -O option",
110static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor,
false,
113 cl::desc(
"Register allocator to use for SGPRs"));
115static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor,
false,
118 cl::desc(
"Register allocator to use for VGPRs"));
121static void initializeDefaultSGPRRegisterAllocatorOnce() {
126 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);
130static void initializeDefaultVGPRRegisterAllocatorOnce() {
135 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);
139static FunctionPass *createBasicSGPRRegisterAllocator() {
143static FunctionPass *createGreedySGPRRegisterAllocator() {
151static FunctionPass *createBasicVGPRRegisterAllocator() {
155static FunctionPass *createGreedyVGPRRegisterAllocator() {
163static SGPRRegisterRegAlloc basicRegAllocSGPR(
164 "basic",
"basic register allocator", createBasicSGPRRegisterAllocator);
165static SGPRRegisterRegAlloc greedyRegAllocSGPR(
166 "greedy",
"greedy register allocator", createGreedySGPRRegisterAllocator);
168static SGPRRegisterRegAlloc fastRegAllocSGPR(
169 "fast",
"fast register allocator", createFastSGPRRegisterAllocator);
172static VGPRRegisterRegAlloc basicRegAllocVGPR(
173 "basic",
"basic register allocator", createBasicVGPRRegisterAllocator);
174static VGPRRegisterRegAlloc greedyRegAllocVGPR(
175 "greedy",
"greedy register allocator", createGreedyVGPRRegisterAllocator);
177static VGPRRegisterRegAlloc fastRegAllocVGPR(
178 "fast",
"fast register allocator", createFastVGPRRegisterAllocator);
183 cl::desc(
"Run early if-conversion"),
188 cl::desc(
"Run pre-RA exec mask optimizations"),
193 cl::desc(
"Lower GPU ctor / dtors to globals on the device."),
198 "amdgpu-load-store-vectorizer",
199 cl::desc(
"Enable load store vectorizer"),
205 "amdgpu-scalarize-global-loads",
206 cl::desc(
"Enable global load scalarization"),
212 "amdgpu-internalize-symbols",
213 cl::desc(
"Enable elimination of non-kernel functions and unused globals"),
219 "amdgpu-early-inline-all",
220 cl::desc(
"Inline all functions early"),
225 "amdgpu-enable-remove-incompatible-functions",
cl::Hidden,
226 cl::desc(
"Enable removal of functions when they"
227 "use features not supported by the target GPU"),
231 "amdgpu-sdwa-peephole",
236 "amdgpu-dpp-combine",
242 cl::desc(
"Enable AMDGPU Alias Analysis"),
247 "amdgpu-late-structurize",
248 cl::desc(
"Enable late CFG structurization"),
255 "amdgpu-disable-structurizer",
256 cl::desc(
"Disable structurizer for experiments; produces unusable code"),
261 "amdgpu-simplify-libcall",
262 cl::desc(
"Enable amdgpu library simplifications"),
267 "amdgpu-ir-lower-kernel-arguments",
268 cl::desc(
"Lower kernel argument loads in IR pass"),
273 "amdgpu-reassign-regs",
274 cl::desc(
"Enable register reassign optimizations on gfx10+"),
279 "amdgpu-opt-vgpr-liverange",
280 cl::desc(
"Enable VGPR liverange optimizations for if-else structure"),
284 "amdgpu-atomic-optimizer-strategy",
285 cl::desc(
"Select DPP or Iterative strategy for scan"),
288 clEnumValN(ScanOptions::DPP,
"DPP",
"Use DPP operations for scan"),
289 clEnumValN(ScanOptions::Iterative,
"Iterative",
290 "Use Iterative approach for scan"),
291 clEnumValN(ScanOptions::None,
"None",
"Disable atomic optimizer")));
295 "amdgpu-mode-register",
296 cl::desc(
"Enable mode register pass"),
303 cl::desc(
"Enable s_singleuse_vdst insertion"),
309 cl::desc(
"Enable s_delay_alu insertion"),
315 cl::desc(
"Enable VOPD, dual issue of VALU in wave32"),
322 cl::desc(
"Enable machine DCE inside regalloc"));
329 "amdgpu-scalar-ir-passes",
330 cl::desc(
"Enable scalar IR passes"),
335 "amdgpu-enable-structurizer-workarounds",
340 "amdgpu-enable-lower-module-lds",
cl::desc(
"Enable lower module lds pass"),
345 "amdgpu-enable-pre-ra-optimizations",
350 "amdgpu-enable-promote-kernel-arguments",
351 cl::desc(
"Enable promotion of flat kernel pointer arguments to global"),
355 "amdgpu-enable-image-intrinsic-optimizer",
361 cl::desc(
"Enable loop data prefetch on AMDGPU"),
365 "amdgpu-enable-max-ilp-scheduling-strategy",
366 cl::desc(
"Enable scheduling strategy to maximize ILP for a single wave."),
370 "amdgpu-enable-rewrite-partial-reg-uses",
375 "amdgpu-enable-hipstdpar",
376 cl::desc(
"Enable HIP Standard Parallelism Offload support"),
cl::init(
false),
461 return std::make_unique<AMDGPUTargetObjectFile>();
474 if (ST.shouldClusterStores())
496 if (ST.shouldClusterStores())
512 if (ST.shouldClusterStores())
524 "Run GCN scheduler to maximize occupancy",
532 "gcn-iterative-max-occupancy-experimental",
533 "Run GCN scheduler to maximize occupancy (experimental)",
537 "gcn-iterative-minreg",
538 "Run GCN iterative scheduler for minimal register usage (experimental)",
543 "Run GCN iterative scheduler for ILP scheduling (experimental)",
549 return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
550 "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1";
559 return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
560 "-p7:160:256:256:32-p8:128:128-p9:192:256:256:32-i64:64-v16:16-v24:32-"
562 "128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-"
587 std::optional<Reloc::Model> RM,
588 std::optional<CodeModel::Model> CM,
611 Attribute GPUAttr =
F.getFnAttribute(
"target-cpu");
616 Attribute FSAttr =
F.getFnAttribute(
"target-features");
624 if (
const Function *
F = dyn_cast<Function>(&GV))
625 return F->isDeclaration() ||
F->getName().starts_with(
"__asan_") ||
626 F->getName().starts_with(
"__sanitizer_") ||
663#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"
753 unsigned DestAS)
const {
759 const auto *LD = dyn_cast<LoadInst>(V);
764 assert(V->getType()->isPointerTy() &&
767 const auto *
Ptr = LD->getPointerOperand();
777std::pair<const Value *, unsigned>
779 if (
auto *II = dyn_cast<IntrinsicInst>(V)) {
780 switch (II->getIntrinsicID()) {
781 case Intrinsic::amdgcn_is_shared:
783 case Intrinsic::amdgcn_is_private:
788 return std::pair(
nullptr, -1);
795 const_cast<Value *
>(V),
797 m_Not(m_Intrinsic<Intrinsic::amdgcn_is_private>(
801 return std::pair(
nullptr, -1);
821 Module &M,
unsigned NumParts,
822 function_ref<
void(std::unique_ptr<Module> MPart)> ModuleCallback)
const {
834 std::optional<Reloc::Model> RM,
835 std::optional<CodeModel::Model> CM,
847 auto &
I = SubtargetMap[SubtargetKey];
853 I = std::make_unique<GCNSubtarget>(
TargetTriple, GPU, FS, *
this);
883 setRequiresCodeGenSCCOrder(
true);
888 return getTM<GCNTargetMachine>();
897 C, std::make_unique<PostGenericScheduler>(
C),
901 if (
ST.shouldClusterStores())
906 if (isPassEnabled(
EnableVOPD, CodeGenOptLevel::Less))
911 bool addPreISel()
override;
912 void addMachineSSAOptimization()
override;
913 bool addILPOpts()
override;
914 bool addInstSelector()
override;
915 bool addIRTranslator()
override;
916 void addPreLegalizeMachineIR()
override;
917 bool addLegalizeMachineIR()
override;
918 void addPreRegBankSelect()
override;
919 bool addRegBankSelect()
override;
920 void addPreGlobalInstructionSelect()
override;
921 bool addGlobalInstructionSelect()
override;
922 void addFastRegAlloc()
override;
923 void addOptimizedRegAlloc()
override;
927 FunctionPass *createRegAllocPass(
bool Optimized)
override;
929 bool addRegAssignAndRewriteFast()
override;
930 bool addRegAssignAndRewriteOptimized()
override;
932 void addPreRegAlloc()
override;
933 bool addPreRewrite()
override;
934 void addPostRegAlloc()
override;
935 void addPreSched2()
override;
936 void addPreEmitPass()
override;
1139 if (ST.shouldClusterStores())
1147 return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
1158 if (ST.enableSIScheduler())
1167bool GCNPassConfig::addPreISel() {
1202void GCNPassConfig::addMachineSSAOptimization() {
1226bool GCNPassConfig::addILPOpts() {
1234bool GCNPassConfig::addInstSelector() {
1241bool GCNPassConfig::addIRTranslator() {
1246void GCNPassConfig::addPreLegalizeMachineIR() {
1252bool GCNPassConfig::addLegalizeMachineIR() {
1257void GCNPassConfig::addPreRegBankSelect() {
1263bool GCNPassConfig::addRegBankSelect() {
1268void GCNPassConfig::addPreGlobalInstructionSelect() {
1273bool GCNPassConfig::addGlobalInstructionSelect() {
1278void GCNPassConfig::addPreRegAlloc() {
1284void GCNPassConfig::addFastRegAlloc() {
1298void GCNPassConfig::addOptimizedRegAlloc() {
1334bool GCNPassConfig::addPreRewrite() {
1341FunctionPass *GCNPassConfig::createSGPRAllocPass(
bool Optimized) {
1344 initializeDefaultSGPRRegisterAllocatorOnce);
1356FunctionPass *GCNPassConfig::createVGPRAllocPass(
bool Optimized) {
1359 initializeDefaultVGPRRegisterAllocatorOnce);
1366 return createGreedyVGPRRegisterAllocator();
1368 return createFastVGPRRegisterAllocator();
1371FunctionPass *GCNPassConfig::createRegAllocPass(
bool Optimized) {
1376 "-regalloc not supported with amdgcn. Use -sgpr-regalloc and -vgpr-regalloc";
1378bool GCNPassConfig::addRegAssignAndRewriteFast() {
1379 if (!usingDefaultRegAlloc())
1384 addPass(createSGPRAllocPass(
false));
1390 addPass(createVGPRAllocPass(
false));
1396bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
1397 if (!usingDefaultRegAlloc())
1402 addPass(createSGPRAllocPass(
true));
1414 addPass(createVGPRAllocPass(
true));
1424void GCNPassConfig::addPostRegAlloc() {
1431void GCNPassConfig::addPreSched2() {
1437void GCNPassConfig::addPreEmitPass() {
1473 return new GCNPassConfig(*
this, PM);
1485 return SIMachineFunctionInfo::create<SIMachineFunctionInfo>(
1512 if (MFI->Occupancy == 0) {
1520 SourceRange =
RegName.SourceRange;
1533 if (parseOptionalRegister(YamlMFI.
VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))
1536 if (parseOptionalRegister(YamlMFI.
SGPRForEXECCopy, MFI->SGPRForEXECCopy))
1540 MFI->LongBranchReservedReg))
1549 "incorrect register class for field",
RegName.Value,
1550 std::nullopt, std::nullopt);
1551 SourceRange =
RegName.SourceRange;
1555 if (parseRegister(YamlMFI.
ScratchRSrcReg, MFI->ScratchRSrcReg) ||
1560 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&
1561 !AMDGPU::SGPR_128RegClass.contains(MFI->ScratchRSrcReg)) {
1565 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&
1566 !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {
1570 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&
1571 !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {
1577 if (parseRegister(YamlReg, ParsedReg))
1583 auto parseAndCheckArgument = [&](
const std::optional<yaml::SIArgument> &
A,
1586 unsigned SystemSGPRs) {
1591 if (
A->IsRegister) {
1594 SourceRange =
A->RegisterName.SourceRange;
1597 if (!RC.contains(Reg))
1598 return diagnoseRegisterClass(
A->RegisterName);
1606 MFI->NumUserSGPRs += UserSGPRs;
1607 MFI->NumSystemSGPRs += SystemSGPRs;
1612 (parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentBuffer,
1613 AMDGPU::SGPR_128RegClass,
1615 parseAndCheckArgument(YamlMFI.
ArgInfo->DispatchPtr,
1616 AMDGPU::SReg_64RegClass, MFI->ArgInfo.
DispatchPtr,
1618 parseAndCheckArgument(YamlMFI.
ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,
1620 parseAndCheckArgument(YamlMFI.
ArgInfo->KernargSegmentPtr,
1621 AMDGPU::SReg_64RegClass,
1623 parseAndCheckArgument(YamlMFI.
ArgInfo->DispatchID,
1624 AMDGPU::SReg_64RegClass, MFI->ArgInfo.
DispatchID,
1626 parseAndCheckArgument(YamlMFI.
ArgInfo->FlatScratchInit,
1627 AMDGPU::SReg_64RegClass,
1629 parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentSize,
1630 AMDGPU::SGPR_32RegClass,
1632 parseAndCheckArgument(YamlMFI.
ArgInfo->LDSKernelId,
1633 AMDGPU::SGPR_32RegClass,
1635 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDX,
1638 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDY,
1641 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupIDZ,
1644 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkGroupInfo,
1645 AMDGPU::SGPR_32RegClass,
1647 parseAndCheckArgument(YamlMFI.
ArgInfo->PrivateSegmentWaveByteOffset,
1648 AMDGPU::SGPR_32RegClass,
1650 parseAndCheckArgument(YamlMFI.
ArgInfo->ImplicitArgPtr,
1651 AMDGPU::SReg_64RegClass,
1653 parseAndCheckArgument(YamlMFI.
ArgInfo->ImplicitBufferPtr,
1654 AMDGPU::SReg_64RegClass,
1656 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDX,
1657 AMDGPU::VGPR_32RegClass,
1659 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDY,
1660 AMDGPU::VGPR_32RegClass,
1662 parseAndCheckArgument(YamlMFI.
ArgInfo->WorkItemIDZ,
1663 AMDGPU::VGPR_32RegClass,
1667 if (ST.hasIEEEMode())
1669 if (ST.hasDX10ClampMode())
static cl::opt< bool > EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(true))
This is the AMGPU address space based alias analysis pass.
Defines an instruction selector for the AMDGPU target.
static cl::opt< bool > EnableDCEInRA("amdgpu-dce-in-ra", cl::init(true), cl::Hidden, cl::desc("Enable machine DCE inside regalloc"))
static cl::opt< bool, true > EnableLowerModuleLDS("amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"), cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true), cl::Hidden)
static MachineSchedRegistry SISchedRegistry("si", "Run SI's custom scheduler", createSIMachineScheduler)
static ScheduleDAGInstrs * createIterativeILPMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EarlyInlineAll("amdgpu-early-inline-all", cl::desc("Inline all functions early"), cl::init(false), cl::Hidden)
static cl::opt< bool > EnableLowerKernelArguments("amdgpu-ir-lower-kernel-arguments", cl::desc("Lower kernel argument loads in IR pass"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createGCNMaxILPMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnableSDWAPeephole("amdgpu-sdwa-peephole", cl::desc("Enable SDWA peepholer"), cl::init(true))
static MachineSchedRegistry GCNMinRegSchedRegistry("gcn-iterative-minreg", "Run GCN iterative scheduler for minimal register usage (experimental)", createMinRegScheduler)
static cl::opt< bool > EnableImageIntrinsicOptimizer("amdgpu-enable-image-intrinsic-optimizer", cl::desc("Enable image intrinsic optimizer pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableSIModeRegisterPass("amdgpu-mode-register", cl::desc("Enable mode register pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableDPPCombine("amdgpu-dpp-combine", cl::desc("Enable DPP combiner"), cl::init(true))
static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry("gcn-iterative-max-occupancy-experimental", "Run GCN scheduler to maximize occupancy (experimental)", createIterativeGCNMaxOccupancyMachineScheduler)
static cl::opt< bool > EnableSetWavePriority("amdgpu-set-wave-priority", cl::desc("Adjust wave priority"), cl::init(false), cl::Hidden)
static cl::opt< bool > LowerCtorDtor("amdgpu-lower-global-ctor-dtor", cl::desc("Lower GPU ctor / dtors to globals on the device."), cl::init(true), cl::Hidden)
static cl::opt< bool, true > DisableStructurizer("amdgpu-disable-structurizer", cl::desc("Disable structurizer for experiments; produces unusable code"), cl::location(AMDGPUTargetMachine::DisableStructurizer), cl::ReallyHidden)
static cl::opt< bool > OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden, cl::desc("Run pre-RA exec mask optimizations"), cl::init(true))
static cl::opt< bool > EnablePromoteKernelArguments("amdgpu-enable-promote-kernel-arguments", cl::desc("Enable promotion of flat kernel pointer arguments to global"), cl::Hidden, cl::init(true))
static cl::opt< bool > EnableRewritePartialRegUses("amdgpu-enable-rewrite-partial-reg-uses", cl::desc("Enable rewrite partial reg uses pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableLibCallSimplify("amdgpu-simplify-libcall", cl::desc("Enable amdgpu library simplifications"), cl::init(true), cl::Hidden)
static MachineSchedRegistry GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp", createGCNMaxILPMachineScheduler)
static cl::opt< bool > InternalizeSymbols("amdgpu-internalize-symbols", cl::desc("Enable elimination of non-kernel functions and unused globals"), cl::init(false), cl::Hidden)
static LLVM_READNONE StringRef getGPUOrDefault(const Triple &TT, StringRef GPU)
static Reloc::Model getEffectiveRelocModel(std::optional< Reloc::Model > RM)
static cl::opt< bool > EnableStructurizerWorkarounds("amdgpu-enable-structurizer-workarounds", cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden, cl::desc("Enable AMDGPU Alias Analysis"), cl::init(true))
static Expected< ScanOptions > parseAMDGPUAtomicOptimizerStrategy(StringRef Params)
static ScheduleDAGInstrs * createMinRegScheduler(MachineSchedContext *C)
static cl::opt< bool, true > LateCFGStructurize("amdgpu-late-structurize", cl::desc("Enable late CFG structurization"), cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG), cl::Hidden)
static cl::opt< bool > EnableHipStdPar("amdgpu-enable-hipstdpar", cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false), cl::Hidden)
static cl::opt< bool > EnableInsertDelayAlu("amdgpu-enable-delay-alu", cl::desc("Enable s_delay_alu insertion"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnableLoadStoreVectorizer("amdgpu-load-store-vectorizer", cl::desc("Enable load store vectorizer"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableMaxIlpSchedStrategy("amdgpu-enable-max-ilp-scheduling-strategy", cl::desc("Enable scheduling strategy to maximize ILP for a single wave."), cl::Hidden, cl::init(false))
static bool mustPreserveGV(const GlobalValue &GV)
Predicate for Internalize pass.
static cl::opt< bool > EnableLoopPrefetch("amdgpu-loop-prefetch", cl::desc("Enable loop data prefetch on AMDGPU"), cl::Hidden, cl::init(false))
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget()
static cl::opt< bool > EnableInsertSingleUseVDST("amdgpu-enable-single-use-vdst", cl::desc("Enable s_singleuse_vdst insertion"), cl::init(false), cl::Hidden)
static cl::opt< bool > RemoveIncompatibleFunctions("amdgpu-enable-remove-incompatible-functions", cl::Hidden, cl::desc("Enable removal of functions when they" "use features not supported by the target GPU"), cl::init(true))
static cl::opt< bool > EnableScalarIRPasses("amdgpu-scalar-ir-passes", cl::desc("Enable scalar IR passes"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableRegReassign("amdgpu-reassign-regs", cl::desc("Enable register reassign optimizations on gfx10+"), cl::init(true), cl::Hidden)
static cl::opt< bool > OptVGPRLiveRange("amdgpu-opt-vgpr-liverange", cl::desc("Enable VGPR liverange optimizations for if-else structure"), cl::init(true), cl::Hidden)
static ScheduleDAGInstrs * createSIMachineScheduler(MachineSchedContext *C)
static cl::opt< bool > EnablePreRAOptimizations("amdgpu-enable-pre-ra-optimizations", cl::desc("Enable Pre-RA optimizations pass"), cl::init(true), cl::Hidden)
static cl::opt< ScanOptions > AMDGPUAtomicOptimizerStrategy("amdgpu-atomic-optimizer-strategy", cl::desc("Select DPP or Iterative strategy for scan"), cl::init(ScanOptions::Iterative), cl::values(clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"), clEnumValN(ScanOptions::Iterative, "Iterative", "Use Iterative approach for scan"), clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")))
static cl::opt< bool > EnableVOPD("amdgpu-enable-vopd", cl::desc("Enable VOPD, dual issue of VALU in wave32"), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(false))
static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)
static MachineSchedRegistry GCNILPSchedRegistry("gcn-iterative-ilp", "Run GCN iterative scheduler for ILP scheduling (experimental)", createIterativeILPMachineScheduler)
static cl::opt< bool > ScalarizeGlobal("amdgpu-scalarize-global-loads", cl::desc("Enable global load scalarization"), cl::init(true), cl::Hidden)
static const char RegAllocOptNotSupportedMessage[]
static MachineSchedRegistry GCNMaxOccupancySchedRegistry("gcn-max-occupancy", "Run GCN scheduler to maximize occupancy", createGCNMaxOccupancyMachineScheduler)
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file declares the AMDGPU-specific subclass of TargetLoweringObjectFile.
Provides passes to inlining "always_inline" functions.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This header provides classes for managing passes over SCCs of the call graph.
Provides analysis for continuously CSEing during GISel passes.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define LLVM_EXTERNAL_VISIBILITY
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines the class GCNIterativeScheduler, which uses an iterative approach to find a best sc...
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
AcceleratorCodeSelection - Identify all functions reachable from a kernel, removing those that are un...
This file declares the IRTranslator pass.
static std::string computeDataLayout()
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
PassInstrumentationCallbacks PIC
PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)
This header defines various interfaces for pass management in LLVM.
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file describes the interface of the MachineFunctionPass responsible for assigning the generic vi...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI Machine Scheduler interface.
static FunctionPass * useDefaultRegisterAllocator()
-regalloc=... command line option.
Target-Independent Code Generator Pass Configuration Options pass.
static std::unique_ptr< TargetLoweringObjectFile > createTLOF()
A manager for alias analyses.
void registerFunctionAnalysis()
Register a specific AA result.
void addAAResult(AAResultT &AAResult)
Register a specific AA result.
Legacy wrapper pass to provide the AMDGPUAAResult object.
Analysis pass providing a never-invalidated alias analysis result.
uint32_t getLDSSize() const
AMDGPUTargetMachine & getAMDGPUTargetMachine() const
std::unique_ptr< CSEConfigBase > getCSEConfig() const override
Returns the CSEConfig object to use for the current optimization level.
ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override
Create an instance of ScheduleDAGInstrs to be run within the standard MachineScheduler pass for this ...
bool isPassEnabled(const cl::opt< bool > &Opt, CodeGenOptLevel Level=CodeGenOptLevel::Default) const
Check if a pass is enabled given Opt option.
bool addPreISel() override
Methods with trivial inline returns are convenient points in the common codegen pass pipeline where t...
bool addInstSelector() override
addInstSelector - This method should install an instruction selector pass, which converts from LLVM c...
bool addGCPasses() override
addGCPasses - Add late codegen passes that analyze code for garbage collection.
void addStraightLineScalarOptimizationPasses()
AMDGPUPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
void addIRPasses() override
Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...
void addEarlyCSEOrGVNPass()
void addCodeGenPrepare() override
Add pass to prepare the LLVM IR for code generation.
static int64_t getNullPointerValue(unsigned AddrSpace)
Get the integer value of a null pointer in the given address space.
unsigned getAddressSpaceForPseudoSourceKind(unsigned Kind) const override
getAddressSpaceForPseudoSourceKind - Given the kind of memory (e.g.
const TargetSubtargetInfo * getSubtargetImpl() const
void registerDefaultAliasAnalyses(AAManager &) override
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
~AMDGPUTargetMachine() override
std::pair< const Value *, unsigned > getPredicatedAddrSpace(const Value *V) const override
If the specified predicate checks whether a generic pointer falls within a specified address space,...
StringRef getFeatureString(const Function &F) const
static bool EnableLateStructurizeCFG
static bool EnableFunctionCalls
AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL)
bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
Error buildCodeGenPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opts, PassInstrumentationCallbacks *PIC) override
void registerPassBuilderCallbacks(PassBuilder &PB, bool PopulateClassToPassNames) override
Allow the target to modify the pass pipeline.
bool splitModule(Module &M, unsigned NumParts, function_ref< void(std::unique_ptr< Module > MPart)> ModuleCallback) const override
Entry point for module splitting.
static bool EnableLowerModuleLDS
StringRef getGPUName(const Function &F) const
unsigned getAssumedAddrSpace(const Value *V) const override
If the specified generic pointer could be assumed as a pointer to a specific address space,...
static bool DisableStructurizer
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
Allocate memory in an ever growing pool, as if by bump-pointer.
Error buildPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const
void removeDeadConstantUsers() const
If there are any dead constant users dangling off of this constant, remove them.
This pass is required by interprocedural register allocation.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
FunctionPass class - This class is used to implement most global optimizations.
@ SCHEDULE_LEGACYMAXOCCUPANCY
const SIRegisterInfo * getRegisterInfo() const override
TargetTransformInfo getTargetTransformInfo(const Function &F) const override
Get a TargetTransformInfo implementation for the target.
void registerMachineRegisterInfoCallback(MachineFunction &MF) const override
bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) const override
Parse out the target's MachineFunctionInfo from the YAML reprsentation.
yaml::MachineFunctionInfo * convertFuncInfoToYAML(const MachineFunction &MF) const override
Allocate and initialize an instance of the YAML representation of the MachineFunctionInfo.
yaml::MachineFunctionInfo * createDefaultFuncInfoYAML() const override
Allocate and return a default initialized instance of the YAML representation for the MachineFunction...
TargetPassConfig * createPassConfig(PassManagerBase &PM) override
Create a pass configuration object to be used by addPassToEmitX methods for generating a pipeline of ...
GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL, bool JIT)
MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override
Create the target's instance of MachineFunctionInfo.
Pass to remove unused function declarations.
This pass is responsible for selecting generic machine instructions to target-specific instructions.
A pass that internalizes all functions and variables other than those that must be preserved accordin...
This class describes a target machine that is implemented with the LLVM target-independent code gener...
This pass implements the localization mechanism described at the top of this file.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void addDelegate(Delegate *delegate)
MachineSchedRegistry provides a selection of available machine instruction schedulers.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
A Module instance is used to store all the information related to an LLVM module.
static const OptimizationLevel O0
Disable as many optimizations as possible.
unsigned getSpeedupLevel() const
static const OptimizationLevel O1
Optimize quickly without destroying debuggability.
This class provides access to building LLVM's passes.
void registerPipelineEarlySimplificationEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerPipelineStartEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerPeepholeEPCallback(const std::function< void(FunctionPassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerCGSCCOptimizerLateEPCallback(const std::function< void(CGSCCPassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
void registerFullLinkTimeOptimizationLastEPCallback(const std::function< void(ModulePassManager &, OptimizationLevel)> &C)
Register a callback for a default optimizer pipeline extension point.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
LLVM_ATTRIBUTE_MINSIZE void addPass(PassT &&Pass)
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
@ ExternalSymbolCallEntry
MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override
Create the target's instance of MachineFunctionInfo.
RegisterPassParser class - Handle the addition of new machine passes.
RegisterRegAllocBase class - Track the registration of register allocators.
FunctionPass *(*)() FunctionPassCtor
Wrapper class representing virtual and physical registers.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void reserveWWMRegister(Register Reg)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
A ScheduleDAG for scheduling lists of MachineInstr.
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
const TargetInstrInfo * TII
Target instruction information.
const TargetRegisterInfo * TRI
Target processor register info.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Triple TargetTriple
Triple string, CPU name, and target feature strings the TargetMachine instance is created with.
const Triple & getTargetTriple() const
const MCSubtargetInfo * getMCSubtargetInfo() const
StringRef getTargetFeatureString() const
StringRef getTargetCPU() const
std::unique_ptr< const MCSubtargetInfo > STI
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
std::unique_ptr< const MCRegisterInfo > MRI
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Target-Independent Code Generator Pass Configuration Options.
virtual void addCodeGenPrepare()
Add pass to prepare the LLVM IR for code generation.
virtual bool addILPOpts()
Add passes that optimize instruction level parallelism for out-of-order targets.
virtual void addPostRegAlloc()
This method may be implemented by targets that want to run passes after register allocation pass pipe...
CodeGenOptLevel getOptLevel() const
virtual void addOptimizedRegAlloc()
addOptimizedRegAlloc - Add passes related to register allocation.
virtual void addIRPasses()
Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...
virtual void addFastRegAlloc()
addFastRegAlloc - Add the minimum set of target-independent passes that are required for fast registe...
virtual void addMachineSSAOptimization()
addMachineSSAOptimization - Add standard passes that optimize machine instructions in SSA form.
void disablePass(AnalysisID PassID)
Allow the target to disable a specific standard pass by default.
AnalysisID addPass(AnalysisID PassID)
Utilities for targets to add passes to the pass manager.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ArchType getArch() const
Get the parsed architecture type of this triple.
LLVM Value Representation.
An efficient, type-erasing, non-owning reference to a callable.
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
An abstract base class for streams implementations that also support a pwrite operation.
Interfaces for registering analysis passes, producing common pass manager configurations,...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
bool isFlatGlobalAddrSpace(unsigned AS)
bool isEntryFunctionCC(CallingConv::ID CC)
@ C
The default llvm calling convention, compatible with C.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
bool match(Val *V, const Pattern &P)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createFlattenCFGPass()
void initializeSIFormMemoryClausesPass(PassRegistry &)
char & SIPreAllocateWWMRegsID
FunctionPass * createFastRegisterAllocator()
FastRegisterAllocation Pass - This pass register allocates as fast as possible.
char & EarlyMachineLICMID
This pass performs loop invariant code motion on machine instructions.
ImmutablePass * createAMDGPUAAWrapperPass()
char & PostRAHazardRecognizerID
PostRAHazardRecognizer - This pass runs the post-ra hazard recognizer.
FunctionPass * createAMDGPUSetWavePriorityPass()
void initializeAMDGPUInsertSingleUseVDSTPass(PassRegistry &)
void initializeGCNCreateVOPDPass(PassRegistry &)
ModulePass * createAMDGPUOpenCLEnqueuedBlockLoweringPass()
char & GCNPreRAOptimizationsID
char & GCLoweringID
GCLowering Pass - Used by gc.root to perform its default lowering operations.
void initializeGCNPreRAOptimizationsPass(PassRegistry &)
Pass * createLoadStoreVectorizerPass()
Create a legacy pass manager instance of the LoadStoreVectorizer pass.
void initializeGCNRewritePartialRegUsesPass(llvm::PassRegistry &)
void initializeAMDGPUAttributorLegacyPass(PassRegistry &)
FunctionPass * createSIModeRegisterPass()
FunctionPass * createGreedyRegisterAllocator()
Greedy register allocation pass - This pass implements a global register allocator for optimized buil...
void initializeAMDGPUAAWrapperPassPass(PassRegistry &)
ModulePass * createAMDGPULowerBufferFatPointersPass()
void initializeR600ClauseMergePassPass(PassRegistry &)
void initializeSIModeRegisterPass(PassRegistry &)
ModulePass * createAMDGPUCtorDtorLoweringLegacyPass()
void initializeSIOptimizeVGPRLiveRangePass(PassRegistry &)
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
void initializeAMDGPULateCodeGenPreparePass(PassRegistry &)
void initializeAMDGPURewriteUndefForPHILegacyPass(PassRegistry &)
FunctionPass * createAMDGPUPreLegalizeCombiner(bool IsOptNone)
char & GCNRewritePartialRegUsesID
FunctionPass * createAMDGPUPostLegalizeCombiner(bool IsOptNone)
void initializeAMDGPUAnnotateUniformValuesPass(PassRegistry &)
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
void initializeSIShrinkInstructionsPass(PassRegistry &)
void initializeGCNPreRALongBranchRegPass(PassRegistry &)
char & SILoadStoreOptimizerID
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation(AMDGPU::SchedulingPhase Phase)
Phase specifes whether or not this is a reentry into the IGroupLPDAGMutation.
void initializeAMDGPUDAGToDAGISelLegacyPass(PassRegistry &)
FunctionPass * createNaryReassociatePass()
char & PatchableFunctionID
This pass implements the "patchable-function" attribute.
char & PostRASchedulerID
PostRAScheduler - This pass performs post register allocation scheduling.
void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &)
void initializeR600PacketizerPass(PassRegistry &)
std::unique_ptr< ScheduleDAGMutation > createVOPDPairingMutation()
ModulePass * createAMDGPUAlwaysInlinePass(bool GlobalOpt=true)
void initializeSIPreEmitPeepholePass(PassRegistry &)
char & SILowerWWMCopiesID
void initializeSIFixVGPRCopiesPass(PassRegistry &)
void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &)
std::unique_ptr< CSEConfigBase > getStandardCSEConfigForOpt(CodeGenOptLevel Level)
Target & getTheR600Target()
The target for R600 GPUs.
char & MachineSchedulerID
MachineScheduler - This pass schedules machine instructions.
Pass * createStructurizeCFGPass(bool SkipUniformRegions=false)
When SkipUniformRegions is true the structizer will not structurize regions that only contain uniform...
void initializeAMDGPURemoveIncompatibleFunctionsPass(PassRegistry &)
void initializeSILowerWWMCopiesPass(PassRegistry &)
void initializeGCNNSAReassignPass(PassRegistry &)
char & PostMachineSchedulerID
PostMachineScheduler - This pass schedules machine instructions postRA.
void initializeSIInsertWaitcntsPass(PassRegistry &)
char & AMDGPUInsertSingleUseVDSTID
ScheduleDAGMILive * createGenericSchedLive(MachineSchedContext *C)
Create the standard converging machine scheduler.
char & SIFormMemoryClausesID
void initializeAMDGPULowerModuleLDSLegacyPass(PassRegistry &)
void initializeAMDGPUCtorDtorLoweringLegacyPass(PassRegistry &)
void initializeAMDGPURegBankCombinerPass(PassRegistry &)
void initializeSILoadStoreOptimizerPass(PassRegistry &)
void initializeSILateBranchLoweringPass(PassRegistry &)
void initializeSIPeepholeSDWAPass(PassRegistry &)
char & AMDGPUUnifyDivergentExitNodesID
FunctionPass * createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy)
char & ShadowStackGCLoweringID
ShadowStackGCLowering - Implements the custom lowering mechanism used by the shadow stack GC.
void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &)
void initializeAMDGPUExternalAAWrapperPass(PassRegistry &)
void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &)
char & AMDGPUPerfHintAnalysisID
char & SILowerSGPRSpillsID
CodeModel::Model getEffectiveCodeModel(std::optional< CodeModel::Model > CM, CodeModel::Model Default)
Helper method for getting the code model, returning Default if CM does not have a value.
char & SILateBranchLoweringPassID
char & BranchRelaxationPassID
BranchRelaxation - This pass replaces branches that need to jump further than is supported by a branc...
FunctionPass * createSinkingPass()
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false, bool NoRerun=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
FunctionPass * createSIShrinkInstructionsPass()
void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &)
CodeGenFileType
These enums are meant to be passed into addPassesToEmitFile to indicate what type of file to emit,...
void initializeSIPostRABundlerPass(PassRegistry &)
void initializeAMDGPUPromoteAllocaToVectorPass(PassRegistry &)
Pass * createAMDGPUAttributorLegacyPass()
void initializeSIWholeQuadModePass(PassRegistry &)
std::unique_ptr< ScheduleDAGMutation > createStoreClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)
If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...
FunctionPass * createLoopDataPrefetchPass()
FunctionPass * createAMDGPULowerKernelArgumentsPass()
char & AMDGPUInsertDelayAluID
Pass * createAMDGPUAnnotateKernelFeaturesPass()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
std::unique_ptr< ScheduleDAGMutation > createAMDGPUMacroFusionDAGMutation()
Note that you have to add: DAG.addMutation(createAMDGPUMacroFusionDAGMutation()); to AMDGPUPassConfig...
char & StackMapLivenessID
StackMapLiveness - This pass analyses the register live-out set of stackmap/patchpoint intrinsics and...
char & SIOptimizeVGPRLiveRangeID
FunctionPass * createUnifyLoopExitsPass()
char & SIOptimizeExecMaskingPreRAID
FunctionPass * createFixIrreduciblePass()
char & FuncletLayoutID
This pass lays out funclets contiguously.
void initializeSIInsertHardClausesPass(PassRegistry &)
char & DetectDeadLanesID
This pass adds dead/undef flags after analyzing subregister lanes.
void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &)
CodeGenOptLevel
Code generation optimization level.
void initializeSIAnnotateControlFlowPass(PassRegistry &)
ModulePass * createAMDGPUPrintfRuntimeBinding()
void initializeSIMemoryLegalizerPass(PassRegistry &)
Pass * createAlwaysInlinerLegacyPass(bool InsertLifetime=true)
Create a legacy pass manager instance of a pass to inline and remove functions marked as "always_inli...
void initializeR600ControlFlowFinalizerPass(PassRegistry &)
void initializeAMDGPUImageIntrinsicOptimizerPass(PassRegistry &)
FunctionPass * createAMDGPUAnnotateUniformValues()
ModulePass * createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM=nullptr)
void initializeAMDGPUPreLegalizerCombinerPass(PassRegistry &)
FunctionPass * createAMDGPUPromoteAlloca()
FunctionPass * createSeparateConstOffsetFromGEPPass(bool LowerGEP=false)
char & EarlyIfConverterID
EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions.
char & SIPreEmitPeepholeID
ModulePass * createAMDGPURemoveIncompatibleFunctionsPass(const TargetMachine *)
FunctionPass * createSILowerI1CopiesPass()
void initializeGCNRegPressurePrinterPass(PassRegistry &)
void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &)
FunctionPass * createBasicRegisterAllocator()
BasicRegisterAllocation Pass - This pass implements a degenerate global register allocator using the ...
void initializeGlobalISel(PassRegistry &)
Initialize all passes linked into the GlobalISel library.
void initializeSIPreAllocateWWMRegsPass(PassRegistry &)
ModulePass * createR600OpenCLImageTypeLoweringPass()
FunctionPass * createAMDGPUCodeGenPreparePass()
FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a AMDGPU-specific.
Target & getTheGCNTarget()
The target for GCN GPUs.
void initializeAMDGPUAtomicOptimizerPass(PassRegistry &)
char & MachineCSEID
MachineCSE - This pass performs global CSE on machine instructions.
FunctionPass * createAMDGPURegBankCombiner(bool IsOptNone)
std::unique_ptr< ScheduleDAGMutation > createLoadClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)
If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...
void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry &)
void initializeAMDGPUMarkLastScratchLoadPass(PassRegistry &)
char & LiveVariablesID
LiveVariables pass - This pass computes the set of blocks in which each variable is life and sets mac...
void initializeAMDGPUCodeGenPreparePass(PassRegistry &)
FunctionPass * createGVNPass(bool NoMemDepAnalysis=false)
Create a legacy GVN pass.
FunctionPass * createAMDGPURewriteUndefForPHILegacyPass()
void call_once(once_flag &flag, Function &&F, Args &&... ArgList)
Execute the function specified as a parameter once.
void initializeSILowerSGPRSpillsPass(PassRegistry &)
void initializeAMDGPULowerKernelAttributesPass(PassRegistry &)
char & SIInsertHardClausesID
FunctionPass * createAMDGPUMachineCFGStructurizerPass()
void initializeAMDGPUResourceUsageAnalysisPass(PassRegistry &)
void initializeSIFixSGPRCopiesPass(PassRegistry &)
FunctionPass * createInferAddressSpacesPass(unsigned AddressSpace=~0u)
char & VirtRegRewriterID
VirtRegRewriter pass.
void initializeSILowerI1CopiesPass(PassRegistry &)
char & SILowerControlFlowID
FunctionPass * createLowerSwitchPass()
FunctionPass * createVirtRegRewriter(bool ClearVirtRegs=true)
void initializeR600VectorRegMergerPass(PassRegistry &)
ImmutablePass * createExternalAAWrapperPass(std::function< void(Pass &, Function &, AAResults &)> Callback)
A wrapper pass around a callback which can be used to populate the AAResults in the AAResultsWrapperP...
void initializeSIOptimizeExecMaskingPass(PassRegistry &)
FunctionPass * createAMDGPUGlobalISelDivergenceLoweringPass()
FunctionPass * createSIMemoryLegalizerPass()
void initializeSIFoldOperandsPass(PassRegistry &)
void initializeSILowerControlFlowPass(PassRegistry &)
void splitAMDGPUModule(const AMDGPUTargetMachine &TM, Module &M, unsigned N, function_ref< void(std::unique_ptr< Module > MPart)> ModuleCallback)
Splits the module M into N linkable partitions.
char & TwoAddressInstructionPassID
TwoAddressInstruction - This pass reduces two-address instructions to use two operands.
void initializeAMDGPURegBankSelectPass(PassRegistry &)
FunctionPass * createAtomicExpandLegacyPass()
AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...
MCRegisterInfo * createGCNMCRegisterInfo(AMDGPUDwarfFlavour DwarfFlavour)
FunctionPass * createStraightLineStrengthReducePass()
FunctionPass * createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *)
void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry &)
void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &)
FunctionPass * createSIInsertWaitcntsPass()
FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)
void initializeGCNDPPCombinePass(PassRegistry &)
char & PHIEliminationID
PHIElimination - This pass eliminates machine instruction PHI nodes by inserting copy instructions.
bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, Register &Reg, StringRef Src, SMDiagnostic &Error)
FunctionPass * createAMDGPULateCodeGenPreparePass()
char & AMDGPUMarkLastScratchLoadID
char & RenameIndependentSubregsID
This pass detects subregister lanes in a virtual register that are used independently of other lanes ...
std::unique_ptr< ScheduleDAGMutation > createAMDGPUExportClusteringDAGMutation()
void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry &)
void initializeAMDGPUPromoteAllocaPass(PassRegistry &)
void initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(PassRegistry &)
void initializeAMDGPUInsertDelayAluPass(PassRegistry &)
char & SIOptimizeExecMaskingID
void initializeAMDGPUUnifyMetadataPass(PassRegistry &)
FunctionPass * createSIAnnotateControlFlowPass()
Create the annotation pass.
void initializeAMDGPUAlwaysInlinePass(PassRegistry &)
char & DeadMachineInstructionElimID
DeadMachineInstructionElim - This pass removes dead machine instructions.
char & GCNPreRALongBranchRegID
void initializeAMDGPUPromoteKernelArgumentsPass(PassRegistry &)
ArgDescriptor PrivateSegmentBuffer
ArgDescriptor WorkGroupIDY
ArgDescriptor WorkGroupIDZ
ArgDescriptor PrivateSegmentSize
ArgDescriptor ImplicitArgPtr
ArgDescriptor PrivateSegmentWaveByteOffset
ArgDescriptor WorkGroupInfo
ArgDescriptor WorkItemIDZ
ArgDescriptor WorkItemIDY
ArgDescriptor LDSKernelId
ArgDescriptor KernargSegmentPtr
ArgDescriptor WorkItemIDX
ArgDescriptor FlatScratchInit
ArgDescriptor DispatchPtr
ArgDescriptor ImplicitBufferPtr
ArgDescriptor WorkGroupIDX
static ArgDescriptor createStack(unsigned Offset, unsigned Mask=~0u)
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ IEEE
IEEE-754 denormal numbers preserved.
DenormalModeKind Output
Denormal flushing mode for floating point instruction results in the default floating point environme...
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
RegisterTargetMachine - Helper template for registering a target machine implementation,...
bool DX10Clamp
Used by the vector ALU to force DX10-style treatment of NaNs: when set, clamp NaN to zero; otherwise,...
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
bool IEEE
Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
The llvm::once_flag structure.
Targets should override this in a way that mirrors the implementation of llvm::MachineFunctionInfo.
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
StringValue FrameOffsetReg
StringValue LongBranchReservedReg
StringValue VGPRForAGPRCopy
std::optional< SIArgumentInfo > ArgInfo
StringValue ScratchRSrcReg
StringValue StackPtrOffsetReg
bool FP64FP16OutputDenormals
bool FP64FP16InputDenormals
A wrapper around std::string which contains a source range that's being set during parsing.