llvm.org GIT mirror llvm / dd98c46
Revert 303091. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@303098 91177308-0d34-0410-b5e6-96231b3b80d8 Jan Sjodin 3 years ago
7 changed file(s) with 14 addition(s) and 3382 deletion(s). Raw diff Collapse all Expand all
4949 FunctionPass *createSIInsertWaitsPass();
5050 FunctionPass *createSIInsertWaitcntsPass();
5151 FunctionPass *createAMDGPUCodeGenPreparePass(const GCNTargetMachine *TM = nullptr);
52 FunctionPass *createAMDGPUMachineCFGStructurizerPass();
53
54 void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&);
55 extern char &AMDGPUMachineCFGStructurizerID;
5652
5753 ModulePass *createAMDGPUAnnotateKernelFeaturesPass(const TargetMachine *TM = nullptr);
5854 void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &);
+0
-3019
lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp less more
None //===-- AMDGPUMachineCFGStructurizer.cpp - Machine code if conversion pass. -----===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the machine instruction level CFG structurizer pass.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "AMDGPU.h"
14 #include "SIInstrInfo.h"
15 #include "AMDGPUSubtarget.h"
16 #include "llvm/ADT/DenseSet.h"
17 #include "llvm/ADT/PostOrderIterator.h"
18 #include "llvm/ADT/SetVector.h"
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/Analysis/CFG.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineFunctionPass.h"
24 #include "llvm/CodeGen/MachineInstr.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineRegionInfo.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/Passes.h"
29 #include "llvm/IR/DebugLoc.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Target/TargetInstrInfo.h"
32 #include "llvm/Target/TargetLowering.h"
33 #include "llvm/Target/TargetSubtargetInfo.h"
34 #include
35 using namespace llvm;
36
37 #define DEBUG_TYPE "amdgpucfgstructurizer"
38
39 namespace {
40 class PHILinearizeDestIterator;
41
42 class PHILinearize {
43 friend class PHILinearizeDestIterator;
44
45 public:
46 typedef std::pair PHISourceT;
47
48 private:
49 typedef DenseSet PHISourcesT;
50 typedef struct {
51 unsigned DestReg;
52 DebugLoc DL;
53 PHISourcesT Sources;
54 } PHIInfoElementT;
55 typedef SmallPtrSet PHIInfoT;
56 PHIInfoT PHIInfo;
57
58 static unsigned phiInfoElementGetDest(PHIInfoElementT *Info);
59 static void phiInfoElementSetDef(PHIInfoElementT *Info, unsigned NewDef);
60 static DebugLoc phiInfoElementGetDebugLoc(PHIInfoElementT *Info);
61 static PHISourcesT &phiInfoElementGetSources(PHIInfoElementT *Info);
62 static void phiInfoElementAddSource(PHIInfoElementT *Info, unsigned SourceReg,
63 MachineBasicBlock *SourceMBB);
64 static void phiInfoElementRemoveSource(PHIInfoElementT *Info,
65 unsigned SourceReg,
66 MachineBasicBlock *SourceMBB);
67 PHIInfoElementT *findPHIInfoElement(unsigned DestReg);
68 PHIInfoElementT *findPHIInfoElementFromSource(unsigned SourceReg,
69 MachineBasicBlock *SourceMBB);
70
71 public:
72 bool findSourcesFromMBB(MachineBasicBlock *SourceMBB,
73 SmallVector &Sources);
74 void addDest(unsigned DestReg, const DebugLoc &DL);
75 void replaceDef(unsigned OldDestReg, unsigned NewDestReg);
76 void deleteDef(unsigned DestReg);
77 DebugLoc getDebugLoc(unsigned DestReg);
78 void addSource(unsigned DestReg, unsigned SourceReg,
79 MachineBasicBlock *SourceMBB);
80 void removeSource(unsigned DestReg, unsigned SourceReg,
81 MachineBasicBlock *SourceMBB = nullptr);
82 bool findDest(unsigned SourceReg, MachineBasicBlock *SourceMBB,
83 unsigned &DestReg);
84 bool isSource(unsigned Reg, MachineBasicBlock *SourceMBB = nullptr);
85 unsigned getNumSources(unsigned DestReg);
86 void dump(MachineRegisterInfo *MRI);
87 void clear();
88
89 typedef PHISourcesT::iterator source_iterator;
90 typedef PHILinearizeDestIterator dest_iterator;
91
92 dest_iterator dests_begin();
93 dest_iterator dests_end();
94
95 source_iterator sources_begin(unsigned Reg);
96 source_iterator sources_end(unsigned Reg);
97 };
98
99 class PHILinearizeDestIterator {
100 private:
101 PHILinearize::PHIInfoT::iterator Iter;
102
103 public:
104 unsigned operator*() { return PHILinearize::phiInfoElementGetDest(*Iter); }
105 PHILinearizeDestIterator &operator++() {
106 ++Iter;
107 return *this;
108 }
109 bool operator==(const PHILinearizeDestIterator &I) const {
110 return I.Iter == Iter;
111 }
112 bool operator!=(const PHILinearizeDestIterator &I) const {
113 return I.Iter != Iter;
114 }
115
116 PHILinearizeDestIterator(PHILinearize::PHIInfoT::iterator I) : Iter(I) {}
117 };
118
119 unsigned PHILinearize::phiInfoElementGetDest(PHIInfoElementT *Info) {
120 return Info->DestReg;
121 }
122
123 void PHILinearize::phiInfoElementSetDef(PHIInfoElementT *Info,
124 unsigned NewDef) {
125 Info->DestReg = NewDef;
126 }
127
128 DebugLoc PHILinearize::phiInfoElementGetDebugLoc(PHIInfoElementT *Info) {
129 return Info->DL;
130 }
131
132 PHILinearize::PHISourcesT &
133 PHILinearize::phiInfoElementGetSources(PHIInfoElementT *Info) {
134 return Info->Sources;
135 }
136
137 void PHILinearize::phiInfoElementAddSource(PHIInfoElementT *Info,
138 unsigned SourceReg,
139 MachineBasicBlock *SourceMBB) {
140 // Assertion ensures we don't use the same SourceMBB for the
141 // sources, because we cannot have different registers with
142 // identical predecessors, but we can have the same register for
143 // multiple predecessors.
144 for (auto SI : phiInfoElementGetSources(Info)) {
145 assert((SI.second != SourceMBB || SourceReg == SI.first));
146 }
147
148 phiInfoElementGetSources(Info).insert(PHISourceT(SourceReg, SourceMBB));
149 }
150
151 void PHILinearize::phiInfoElementRemoveSource(PHIInfoElementT *Info,
152 unsigned SourceReg,
153 MachineBasicBlock *SourceMBB) {
154 auto &Sources = phiInfoElementGetSources(Info);
155 SmallVector ElimiatedSources;
156 for (auto SI : Sources) {
157 if (SI.first == SourceReg &&
158 (SI.second == nullptr || SI.second == SourceMBB)) {
159 ElimiatedSources.push_back(PHISourceT(SI.first, SI.second));
160 }
161 }
162
163 for (auto &Source : ElimiatedSources) {
164 Sources.erase(Source);
165 }
166 }
167
168 PHILinearize::PHIInfoElementT *
169 PHILinearize::findPHIInfoElement(unsigned DestReg) {
170 for (auto I : PHIInfo) {
171 if (phiInfoElementGetDest(I) == DestReg) {
172 return I;
173 }
174 }
175 return nullptr;
176 }
177
178 PHILinearize::PHIInfoElementT *
179 PHILinearize::findPHIInfoElementFromSource(unsigned SourceReg,
180 MachineBasicBlock *SourceMBB) {
181 for (auto I : PHIInfo) {
182 for (auto SI : phiInfoElementGetSources(I)) {
183 if (SI.first == SourceReg &&
184 (SI.second == nullptr || SI.second == SourceMBB)) {
185 return I;
186 }
187 }
188 }
189 return nullptr;
190 }
191
192 bool PHILinearize::findSourcesFromMBB(MachineBasicBlock *SourceMBB,
193 SmallVector &Sources) {
194 bool FoundSource = false;
195 for (auto I : PHIInfo) {
196 for (auto SI : phiInfoElementGetSources(I)) {
197 if (SI.second == SourceMBB) {
198 FoundSource = true;
199 Sources.push_back(SI.first);
200 }
201 }
202 }
203 return FoundSource;
204 }
205
206 void PHILinearize::addDest(unsigned DestReg, const DebugLoc &DL) {
207 assert(findPHIInfoElement(DestReg) == nullptr && "Dest already exsists");
208 PHISourcesT EmptySet;
209 PHIInfoElementT *NewElement = new PHIInfoElementT();
210 NewElement->DestReg = DestReg;
211 NewElement->DL = DL;
212 NewElement->Sources = EmptySet;
213 PHIInfo.insert(NewElement);
214 }
215
216 void PHILinearize::replaceDef(unsigned OldDestReg, unsigned NewDestReg) {
217 phiInfoElementSetDef(findPHIInfoElement(OldDestReg), NewDestReg);
218 }
219
220 void PHILinearize::deleteDef(unsigned DestReg) {
221 PHIInfoElementT *InfoElement = findPHIInfoElement(DestReg);
222 PHIInfo.erase(InfoElement);
223 delete InfoElement;
224 }
225
226 DebugLoc PHILinearize::getDebugLoc(unsigned DestReg) {
227 return phiInfoElementGetDebugLoc(findPHIInfoElement(DestReg));
228 }
229
230 void PHILinearize::addSource(unsigned DestReg, unsigned SourceReg,
231 MachineBasicBlock *SourceMBB) {
232 phiInfoElementAddSource(findPHIInfoElement(DestReg), SourceReg, SourceMBB);
233 }
234
235 void PHILinearize::removeSource(unsigned DestReg, unsigned SourceReg,
236 MachineBasicBlock *SourceMBB) {
237 phiInfoElementRemoveSource(findPHIInfoElement(DestReg), SourceReg, SourceMBB);
238 }
239
240 bool PHILinearize::findDest(unsigned SourceReg, MachineBasicBlock *SourceMBB,
241 unsigned &DestReg) {
242 PHIInfoElementT *InfoElement =
243 findPHIInfoElementFromSource(SourceReg, SourceMBB);
244 if (InfoElement != nullptr) {
245 DestReg = phiInfoElementGetDest(InfoElement);
246 return true;
247 }
248 return false;
249 }
250
251 bool PHILinearize::isSource(unsigned Reg, MachineBasicBlock *SourceMBB) {
252 unsigned DestReg;
253 return findDest(Reg, SourceMBB, DestReg);
254 }
255
256 unsigned PHILinearize::getNumSources(unsigned DestReg) {
257 return phiInfoElementGetSources(findPHIInfoElement(DestReg)).size();
258 }
259
260 void PHILinearize::dump(MachineRegisterInfo *MRI) {
261 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
262 DEBUG(dbgs() << "=PHIInfo Start=\n");
263 for (auto PII : this->PHIInfo) {
264 PHIInfoElementT &Element = *PII;
265 DEBUG(dbgs() << "Dest: " << PrintReg(Element.DestReg, TRI)
266 << " Sources: {");
267 for (auto &SI : Element.Sources) {
268 DEBUG(dbgs() << PrintReg(SI.first, TRI) << "(BB#"
269 << SI.second->getNumber() << "),");
270 }
271 DEBUG(dbgs() << "}\n");
272 }
273 DEBUG(dbgs() << "=PHIInfo End=\n");
274 }
275
276 void PHILinearize::clear() { PHIInfo = PHIInfoT(); }
277
278 PHILinearize::dest_iterator PHILinearize::dests_begin() {
279 return PHILinearizeDestIterator(PHIInfo.begin());
280 }
281
282 PHILinearize::dest_iterator PHILinearize::dests_end() {
283 return PHILinearizeDestIterator(PHIInfo.end());
284 }
285
286 PHILinearize::source_iterator PHILinearize::sources_begin(unsigned Reg) {
287 auto InfoElement = findPHIInfoElement(Reg);
288 return phiInfoElementGetSources(InfoElement).begin();
289 }
290 PHILinearize::source_iterator PHILinearize::sources_end(unsigned Reg) {
291 auto InfoElement = findPHIInfoElement(Reg);
292 return phiInfoElementGetSources(InfoElement).end();
293 }
294
295 class RegionMRT;
296 class MBBMRT;
297
298 static unsigned getPHINumInputs(MachineInstr &PHI) {
299 assert(PHI.isPHI());
300 return (PHI.getNumOperands() - 1) / 2;
301 }
302
303 static MachineBasicBlock *getPHIPred(MachineInstr &PHI, unsigned Index) {
304 assert(PHI.isPHI());
305 return PHI.getOperand(Index * 2 + 2).getMBB();
306 }
307
308 static void setPhiPred(MachineInstr &PHI, unsigned Index,
309 MachineBasicBlock *NewPred) {
310 PHI.getOperand(Index * 2 + 2).setMBB(NewPred);
311 }
312
313 static unsigned getPHISourceReg(MachineInstr &PHI, unsigned Index) {
314 assert(PHI.isPHI());
315 return PHI.getOperand(Index * 2 + 1).getReg();
316 }
317
318 static unsigned getPHIDestReg(MachineInstr &PHI) {
319 assert(PHI.isPHI());
320 return PHI.getOperand(0).getReg();
321 }
322
323 class LinearizedRegion {
324 protected:
325 MachineBasicBlock *Entry;
326 // The exit block is part of the region, and is the last
327 // merge block before exiting the region.
328 MachineBasicBlock *Exit;
329 DenseSet LiveOuts;
330 SmallPtrSet MBBs;
331 bool HasLoop;
332 LinearizedRegion *Parent;
333 RegionMRT *RMRT;
334
335 void storeLiveOutReg(MachineBasicBlock *MBB, unsigned Reg,
336 MachineInstr *DefInstr, const MachineRegisterInfo *MRI,
337 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo);
338
339 void storeLiveOutRegRegion(RegionMRT *Region, unsigned Reg,
340 MachineInstr *DefInstr,
341 const MachineRegisterInfo *MRI,
342 const TargetRegisterInfo *TRI,
343 PHILinearize &PHIInfo);
344
345 void storeMBBLiveOuts(MachineBasicBlock *MBB, const MachineRegisterInfo *MRI,
346 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo,
347 RegionMRT *TopRegion);
348
349 void storeLiveOuts(MachineBasicBlock *MBB, const MachineRegisterInfo *MRI,
350 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo);
351
352 void storeLiveOuts(RegionMRT *Region, const MachineRegisterInfo *MRI,
353 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo,
354 RegionMRT *TopRegion = nullptr);
355
356 public:
357 void setRegionMRT(RegionMRT *Region) { RMRT = Region; }
358
359 RegionMRT *getRegionMRT() { return RMRT; }
360
361 void setParent(LinearizedRegion *P) { Parent = P; }
362
363 LinearizedRegion *getParent() { return Parent; }
364
365 void print(raw_ostream &OS, const TargetRegisterInfo *TRI = nullptr);
366
367 void setBBSelectRegIn(unsigned Reg);
368
369 unsigned getBBSelectRegIn();
370
371 void setBBSelectRegOut(unsigned Reg, bool IsLiveOut);
372
373 unsigned getBBSelectRegOut();
374
375 void setHasLoop(bool Value);
376
377 bool getHasLoop();
378
379 void addLiveOut(unsigned VReg);
380
381 void addLiveOuts(LinearizedRegion *LRegion);
382
383 void removeLiveOut(unsigned Reg);
384
385 void replaceLiveOut(unsigned OldReg, unsigned NewReg);
386
387 void replaceRegister(unsigned Register, unsigned NewRegister,
388 MachineRegisterInfo *MRI, bool ReplaceInside,
389 bool ReplaceOutside, bool IncludeLoopPHIs);
390
391 void replaceRegisterInsideRegion(unsigned Register, unsigned NewRegister,
392 bool IncludeLoopPHIs,
393 MachineRegisterInfo *MRI);
394
395 void replaceRegisterOutsideRegion(unsigned Register, unsigned NewRegister,
396 bool IncludeLoopPHIs,
397 MachineRegisterInfo *MRI);
398
399 DenseSet *getLiveOuts();
400
401 void setEntry(MachineBasicBlock *NewEntry);
402
403 MachineBasicBlock *getEntry();
404
405 void setExit(MachineBasicBlock *NewExit);
406
407 MachineBasicBlock *getExit();
408
409 void addMBB(MachineBasicBlock *MBB);
410
411 void addMBBs(LinearizedRegion *InnerRegion);
412
413 bool contains(MachineBasicBlock *MBB);
414
415 bool isLiveOut(unsigned Reg);
416
417 bool hasNoDef(unsigned Reg, MachineRegisterInfo *MRI);
418
419 bool isDefinedInRegion(unsigned Reg, MachineRegisterInfo *MRI);
420
421 void removeFalseRegisterKills(MachineRegisterInfo *MRI);
422
423 void initLiveOut(RegionMRT *Region, const MachineRegisterInfo *MRI,
424 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo);
425
426 LinearizedRegion(MachineBasicBlock *MBB, const MachineRegisterInfo *MRI,
427 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo);
428
429 LinearizedRegion(RegionMRT *Region, const MachineRegisterInfo *MRI,
430 const TargetRegisterInfo *TRI, PHILinearize &PHIInfo);
431
432 LinearizedRegion();
433
434 ~LinearizedRegion();
435 };
436
437 class MRT {
438 protected:
439 RegionMRT *Parent;
440 unsigned BBSelectRegIn;
441 unsigned BBSelectRegOut;
442
443 public:
444 unsigned getBBSelectRegIn() { return BBSelectRegIn; }
445
446 unsigned getBBSelectRegOut() { return BBSelectRegOut; }
447
448 void setBBSelectRegIn(unsigned Reg) { BBSelectRegIn = Reg; }
449
450 void setBBSelectRegOut(unsigned Reg) { BBSelectRegOut = Reg; }
451
452 virtual RegionMRT *getRegionMRT() { return nullptr; }
453
454 virtual MBBMRT *getMBBMRT() { return nullptr; }
455
456 bool isRegion() { return getRegionMRT() != nullptr; }
457
458 bool isMBB() { return getMBBMRT() != nullptr; }
459
460 bool isRoot() { return Parent == nullptr; }
461
462 void setParent(RegionMRT *Region) { Parent = Region; }
463
464 RegionMRT *getParent() { return Parent; }
465
466 static MachineBasicBlock *
467 initializeMRT(MachineFunction &MF, const MachineRegionInfo *RegionInfo,
468 DenseMap &RegionMap);
469
470 static RegionMRT *buildMRT(MachineFunction &MF,
471 const MachineRegionInfo *RegionInfo,
472 const SIInstrInfo *TII,
473 MachineRegisterInfo *MRI);
474
475 virtual void dump(const TargetRegisterInfo *TRI, int depth = 0) = 0;
476
477 void dumpDepth(int depth) {
478 for (int i = depth; i > 0; --i) {
479 dbgs() << " ";
480 }
481 }
482
483 virtual ~MRT() {}
484 };
485
486 class MBBMRT : public MRT {
487 MachineBasicBlock *MBB;
488
489 public:
490 virtual MBBMRT *getMBBMRT() { return this; }
491
492 MachineBasicBlock *getMBB() { return MBB; }
493
494 virtual void dump(const TargetRegisterInfo *TRI, int depth = 0) {
495 dumpDepth(depth);
496 dbgs() << "MBB: " << getMBB()->getNumber();
497 dbgs() << " In: " << PrintReg(getBBSelectRegIn(), TRI);
498 dbgs() << ", Out: " << PrintReg(getBBSelectRegOut(), TRI) << "\n";
499 }
500
501 MBBMRT(MachineBasicBlock *BB) : MBB(BB) {
502 setParent(nullptr);
503 setBBSelectRegOut(0);
504 setBBSelectRegIn(0);
505 }
506 };
507
508 class RegionMRT : public MRT {
509 protected:
510 MachineRegion *Region;
511 LinearizedRegion *LRegion;
512 MachineBasicBlock *Succ;
513
514 SetVector Children;
515
516 public:
517 virtual RegionMRT *getRegionMRT() { return this; }
518
519 void setLinearizedRegion(LinearizedRegion *LinearizeRegion) {
520 LRegion = LinearizeRegion;
521 }
522
523 LinearizedRegion *getLinearizedRegion() { return LRegion; }
524
525 MachineRegion *getMachineRegion() { return Region; }
526
527 unsigned getInnerOutputRegister() {
528 return (*(Children.begin()))->getBBSelectRegOut();
529 }
530
531 void addChild(MRT *Tree) { Children.insert(Tree); }
532
533 SetVector *getChildren() { return &Children; }
534
535 virtual void dump(const TargetRegisterInfo *TRI, int depth = 0) {
536 dumpDepth(depth);
537 dbgs() << "Region: " << (void *)Region;
538 dbgs() << " In: " << PrintReg(getBBSelectRegIn(), TRI);
539 dbgs() << ", Out: " << PrintReg(getBBSelectRegOut(), TRI) << "\n";
540
541 dumpDepth(depth);
542 if (getSucc())
543 dbgs() << "Succ: " << getSucc()->getNumber() << "\n";
544 else
545 dbgs() << "Succ: none \n";
546 for (auto MRTI : Children) {
547 MRTI->dump(TRI, depth + 1);
548 }
549 }
550
551 MRT *getEntryTree() { return Children.back(); }
552
553 MRT *getExitTree() { return Children.front(); }
554
555 MachineBasicBlock *getEntry() {
556 MRT *Tree = Children.back();
557 return (Tree->isRegion()) ? Tree->getRegionMRT()->getEntry()
558 : Tree->getMBBMRT()->getMBB();
559 }
560
561 MachineBasicBlock *getExit() {
562 MRT *Tree = Children.front();
563 return (Tree->isRegion()) ? Tree->getRegionMRT()->getExit()
564 : Tree->getMBBMRT()->getMBB();
565 }
566
567 void setSucc(MachineBasicBlock *MBB) { Succ = MBB; }
568
569 MachineBasicBlock *getSucc() { return Succ; }
570
571 bool contains(MachineBasicBlock *MBB) {
572 for (auto CI : Children) {
573 if (CI->isMBB()) {
574 if (MBB == CI->getMBBMRT()->getMBB()) {
575 return true;
576 }
577 } else {
578 if (CI->getRegionMRT()->contains(MBB)) {
579 return true;
580 } else if (CI->getRegionMRT()->getLinearizedRegion() != nullptr &&
581 CI->getRegionMRT()->getLinearizedRegion()->contains(MBB)) {
582 return true;
583 }
584 }
585 }
586 return false;
587 }
588
589 void replaceLiveOutReg(unsigned Register, unsigned NewRegister) {
590 LinearizedRegion *LRegion = getLinearizedRegion();
591 LRegion->replaceLiveOut(Register, NewRegister);
592 for (auto &CI : Children) {
593 if (CI->isRegion()) {
594 CI->getRegionMRT()->replaceLiveOutReg(Register, NewRegister);
595 }
596 }
597 }
598
599 RegionMRT(MachineRegion *MachineRegion)
600 : Region(MachineRegion), LRegion(nullptr), Succ(nullptr) {
601 setParent(nullptr);
602 setBBSelectRegOut(0);
603 setBBSelectRegIn(0);
604 }
605
606 virtual ~RegionMRT() {
607 if (LRegion) {
608 delete LRegion;
609 }
610
611 for (auto CI : Children) {
612 delete &(*CI);
613 }
614 }
615 };
616
617 static unsigned createBBSelectReg(const SIInstrInfo *TII,
618 MachineRegisterInfo *MRI) {
619 return MRI->createVirtualRegister(TII->getPreferredSelectRegClass(32));
620 }
621
622 MachineBasicBlock *
623 MRT::initializeMRT(MachineFunction &MF, const MachineRegionInfo *RegionInfo,
624 DenseMap &RegionMap) {
625 for (auto &MFI : MF) {
626 MachineBasicBlock *ExitMBB = &MFI;
627 if (ExitMBB->succ_size() == 0) {
628 return ExitMBB;
629 }
630 }
631 llvm_unreachable("CFG has no exit block");
632 return nullptr;
633 }
634
635 RegionMRT *MRT::buildMRT(MachineFunction &MF,
636 const MachineRegionInfo *RegionInfo,
637 const SIInstrInfo *TII, MachineRegisterInfo *MRI) {
638 SmallPtrSet PlacedRegions;
639 DenseMap RegionMap;
640 MachineRegion *TopLevelRegion = RegionInfo->getTopLevelRegion();
641 RegionMRT *Result = new RegionMRT(TopLevelRegion);
642 RegionMap[TopLevelRegion] = Result;
643
644 // Insert the exit block first, we need it to be the merge node
645 // for the top level region.
646 MachineBasicBlock *Exit = initializeMRT(MF, RegionInfo, RegionMap);
647
648 unsigned BBSelectRegIn = createBBSelectReg(TII, MRI);
649 MBBMRT *ExitMRT = new MBBMRT(Exit);
650 RegionMap[RegionInfo->getRegionFor(Exit)]->addChild(ExitMRT);
651 ExitMRT->setBBSelectRegIn(BBSelectRegIn);
652
653 for (auto MBBI : post_order(&(MF.front()))) {
654 MachineBasicBlock *MBB = &(*MBBI);
655
656 // Skip Exit since we already added it
657 if (MBB == Exit) {
658 continue;
659 }
660
661 DEBUG(dbgs() << "Visiting BB#" << MBB->getNumber() << "\n");
662 MBBMRT *NewMBB = new MBBMRT(MBB);
663 MachineRegion *Region = RegionInfo->getRegionFor(MBB);
664
665 // Ensure we have the MRT region
666 if (RegionMap.count(Region) == 0) {
667 RegionMRT *NewMRTRegion = new RegionMRT(Region);
668 RegionMap[Region] = NewMRTRegion;
669
670 // Ensure all parents are in the RegionMap
671 MachineRegion *Parent = Region->getParent();
672 while (RegionMap.count(Parent) == 0) {
673 RegionMRT *NewMRTParent = new RegionMRT(Parent);
674 NewMRTParent->addChild(NewMRTRegion);
675 NewMRTRegion->setParent(NewMRTParent);
676 RegionMap[Parent] = NewMRTParent;
677 NewMRTRegion = NewMRTParent;
678 Parent = Parent->getParent();
679 }
680 RegionMap[Parent]->addChild(NewMRTRegion);
681 NewMRTRegion->setParent(RegionMap[Parent]);
682 }
683
684 // Add MBB to Region MRT
685 RegionMap[Region]->addChild(NewMBB);
686 NewMBB->setParent(RegionMap[Region]);
687 RegionMap[Region]->setSucc(Region->getExit());
688 }
689 return Result;
690 }
691
692 void LinearizedRegion::storeLiveOutReg(MachineBasicBlock *MBB, unsigned Reg,
693 MachineInstr *DefInstr,
694 const MachineRegisterInfo *MRI,
695 const TargetRegisterInfo *TRI,
696 PHILinearize &PHIInfo) {
697 if (TRI->isVirtualRegister(Reg)) {
698 DEBUG(dbgs() << "Considering Register: " << PrintReg(Reg, TRI) << "\n");
699 // If this is a source register to a PHI we are chaining, it
700 // must be live out.
701 if (PHIInfo.isSource(Reg)) {
702 DEBUG(dbgs() << "Add LiveOut (PHI): " << PrintReg(Reg, TRI) << "\n");
703 addLiveOut(Reg);
704 } else {
705 // If this is live out of the MBB
706 for (auto &UI : MRI->use_operands(Reg)) {
707 if (UI.getParent()->getParent() != MBB) {
708 DEBUG(dbgs() << "Add LiveOut (MBB BB#" << MBB->getNumber()
709 << "): " << PrintReg(Reg, TRI) << "\n");
710 addLiveOut(Reg);
711 } else {
712 // If the use is in the same MBB we have to make sure
713 // it is after the def, otherwise it is live out in a loop
714 MachineInstr *UseInstr = UI.getParent();
715 for (MachineBasicBlock::instr_iterator
716 MII = UseInstr->getIterator(),
717 MIE = UseInstr->getParent()->instr_end();
718 MII != MIE; ++MII) {
719 if ((&(*MII)) == DefInstr) {
720 DEBUG(dbgs() << "Add LiveOut (Loop): " << PrintReg(Reg, TRI)
721 << "\n");
722 addLiveOut(Reg);
723 }
724 }
725 }
726 }
727 }
728 }
729 }
730
731 void LinearizedRegion::storeLiveOutRegRegion(RegionMRT *Region, unsigned Reg,
732 MachineInstr *DefInstr,
733 const MachineRegisterInfo *MRI,
734 const TargetRegisterInfo *TRI,
735 PHILinearize &PHIInfo) {
736 if (TRI->isVirtualRegister(Reg)) {
737 DEBUG(dbgs() << "Considering Register: " << PrintReg(Reg, TRI) << "\n");
738 for (auto &UI : MRI->use_operands(Reg)) {
739 if (!Region->contains(UI.getParent()->getParent())) {
740 DEBUG(dbgs() << "Add LiveOut (Region " << (void *)Region
741 << "): " << PrintReg(Reg, TRI) << "\n");
742 addLiveOut(Reg);
743 }
744 }
745 }
746 }
747
748 void LinearizedRegion::storeLiveOuts(MachineBasicBlock *MBB,
749 const MachineRegisterInfo *MRI,
750 const TargetRegisterInfo *TRI,
751 PHILinearize &PHIInfo) {
752 DEBUG(dbgs() << "-Store Live Outs Begin (BB#" << MBB->getNumber() << ")-\n");
753 for (auto &II : *MBB) {
754 for (auto &RI : II.defs()) {
755 storeLiveOutReg(MBB, RI.getReg(), RI.getParent(), MRI, TRI, PHIInfo);
756 }
757 for (auto &IRI : II.implicit_operands()) {
758 if (IRI.isDef()) {
759 storeLiveOutReg(MBB, IRI.getReg(), IRI.getParent(), MRI, TRI, PHIInfo);
760 }
761 }
762 }
763
764 // If we have a successor with a PHI, source coming from this MBB we have to
765 // add the register as live out
766 for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
767 E = MBB->succ_end();
768 SI != E; ++SI) {
769 for (auto &II : *(*SI)) {
770 if (II.isPHI()) {
771 MachineInstr &PHI = II;
772 int numPreds = getPHINumInputs(PHI);
773 for (int i = 0; i < numPreds; ++i) {
774 if (getPHIPred(PHI, i) == MBB) {
775 unsigned PHIReg = getPHISourceReg(PHI, i);
776 DEBUG(dbgs() << "Add LiveOut (PhiSource BB#" << MBB->getNumber()
777 << " -> BB#" << (*SI)->getNumber()
778 << "): " << PrintReg(PHIReg, TRI) << "\n");
779 addLiveOut(PHIReg);
780 }
781 }
782 }
783 }
784 }
785
786 DEBUG(dbgs() << "-Store Live Outs Endn-\n");
787 }
788
789 void LinearizedRegion::storeMBBLiveOuts(MachineBasicBlock *MBB,
790 const MachineRegisterInfo *MRI,
791 const TargetRegisterInfo *TRI,
792 PHILinearize &PHIInfo,
793 RegionMRT *TopRegion) {
794 for (auto &II : *MBB) {
795 for (auto &RI : II.defs()) {
796 storeLiveOutRegRegion(TopRegion, RI.getReg(), RI.getParent(), MRI, TRI,
797 PHIInfo);
798 }
799 for (auto &IRI : II.implicit_operands()) {
800 if (IRI.isDef()) {
801 storeLiveOutRegRegion(TopRegion, IRI.getReg(), IRI.getParent(), MRI,
802 TRI, PHIInfo);
803 }
804 }
805 }
806 }
807
808 void LinearizedRegion::storeLiveOuts(RegionMRT *Region,
809 const MachineRegisterInfo *MRI,
810 const TargetRegisterInfo *TRI,
811 PHILinearize &PHIInfo,
812 RegionMRT *CurrentTopRegion) {
813 MachineBasicBlock *Exit = Region->getSucc();
814
815 RegionMRT *TopRegion =
816 CurrentTopRegion == nullptr ? Region : CurrentTopRegion;
817
818 // Check if exit is end of function, if so, no live outs.
819 if (Exit == nullptr)
820 return;
821
822 auto Children = Region->getChildren();
823 for (auto CI : *Children) {
824 if (CI->isMBB()) {
825 auto MBB = CI->getMBBMRT()->getMBB();
826 storeMBBLiveOuts(MBB, MRI, TRI, PHIInfo, TopRegion);
827 } else {
828 LinearizedRegion *SubRegion = CI->getRegionMRT()->getLinearizedRegion();
829 // We should be limited to only store registers that are live out from the
830 // lineaized region
831 for (auto MBBI : SubRegion->MBBs) {
832 storeMBBLiveOuts(MBBI, MRI, TRI, PHIInfo, TopRegion);
833 }
834 }
835 }
836
837 if (CurrentTopRegion == nullptr) {
838 auto Succ = Region->getSucc();
839 for (auto &II : *Succ) {
840 if (II.isPHI()) {
841 MachineInstr &PHI = II;
842 int numPreds = getPHINumInputs(PHI);
843 for (int i = 0; i < numPreds; ++i) {
844 if (Region->contains(getPHIPred(PHI, i))) {
845 unsigned PHIReg = getPHISourceReg(PHI, i);
846 DEBUG(dbgs() << "Add Region LiveOut (" << (void *)Region
847 << "): " << PrintReg(PHIReg, TRI) << "\n");
848 addLiveOut(PHIReg);
849 }
850 }
851 }
852 }
853 }
854 }
855
856 void LinearizedRegion::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
857 OS << "Linearized Region {";
858 bool IsFirst = true;
859 for (const auto &MBB : MBBs) {
860 if (IsFirst) {
861 IsFirst = false;
862 } else {
863 OS << " ,";
864 }
865 OS << MBB->getNumber();
866 }
867 OS << "} (" << Entry->getNumber() << ", "
868 << (Exit == nullptr ? -1 : Exit->getNumber())
869 << "): In:" << PrintReg(getBBSelectRegIn(), TRI)
870 << " Out:" << PrintReg(getBBSelectRegOut(), TRI) << " {";
871 for (auto &LI : LiveOuts) {
872 OS << PrintReg(LI, TRI) << " ";
873 }
874 OS << "} \n";
875 }
876
877 unsigned LinearizedRegion::getBBSelectRegIn() {
878 return getRegionMRT()->getBBSelectRegIn();
879 }
880
881 unsigned LinearizedRegion::getBBSelectRegOut() {
882 return getRegionMRT()->getBBSelectRegOut();
883 }
884
885 void LinearizedRegion::setHasLoop(bool Value) { HasLoop = Value; }
886
887 bool LinearizedRegion::getHasLoop() { return HasLoop; }
888
889 void LinearizedRegion::addLiveOut(unsigned VReg) { LiveOuts.insert(VReg); }
890
891 void LinearizedRegion::addLiveOuts(LinearizedRegion *LRegion) {
892 DenseSet *RegionLiveOuts = LRegion->getLiveOuts();
893 for (auto R : *RegionLiveOuts) {
894 addLiveOut(R);
895 }
896 }
897
898 void LinearizedRegion::removeLiveOut(unsigned Reg) {
899 if (isLiveOut(Reg))
900 LiveOuts.erase(Reg);
901 }
902
903 void LinearizedRegion::replaceLiveOut(unsigned OldReg, unsigned NewReg) {
904 if (isLiveOut(OldReg)) {
905 removeLiveOut(OldReg);
906 addLiveOut(NewReg);
907 }
908 }
909
910 void LinearizedRegion::replaceRegister(unsigned Register, unsigned NewRegister,
911 MachineRegisterInfo *MRI,
912 bool ReplaceInside, bool ReplaceOutside,
913 bool IncludeLoopPHI) {
914 assert(Register != NewRegister && "Cannot replace a reg with itself");
915
916 DEBUG(dbgs() << "Pepareing to replace register (region): "
917 << PrintReg(Register, MRI->getTargetRegisterInfo()) << " with "
918 << PrintReg(NewRegister, MRI->getTargetRegisterInfo()) << "\n");
919
920 // If we are replacing outside, we also need to update the LiveOuts
921 if (ReplaceOutside &&
922 (isLiveOut(Register) || this->getParent()->isLiveOut(Register))) {
923 LinearizedRegion *Current = this;
924 while (Current != nullptr && Current->getEntry() != nullptr) {
925 DEBUG(dbgs() << "Region before register replace\n");
926 DEBUG(Current->print(dbgs(), MRI->getTargetRegisterInfo()));
927 Current->replaceLiveOut(Register, NewRegister);
928 DEBUG(dbgs() << "Region after register replace\n");
929 DEBUG(Current->print(dbgs(), MRI->getTargetRegisterInfo()));
930 Current = Current->getParent();
931 }
932 }
933
934 for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(Register),
935 E = MRI->reg_end();
936 I != E;) {
937 MachineOperand &O = *I;
938 ++I;
939
940 // We don't rewrite defs.
941 if (O.isDef())
942 continue;
943
944 bool IsInside = contains(O.getParent()->getParent());
945 bool IsLoopPHI = IsInside && (O.getParent()->isPHI() &&
946 O.getParent()->getParent() == getEntry());
947 bool ShouldReplace = (IsInside && ReplaceInside) ||
948 (!IsInside && ReplaceOutside) ||
949 (IncludeLoopPHI && IsLoopPHI);
950 if (ShouldReplace) {
951
952 if (TargetRegisterInfo::isPhysicalRegister(NewRegister)) {
953 DEBUG(dbgs() << "Trying to substitute physical register: "
954 << PrintReg(NewRegister, MRI->getTargetRegisterInfo())
955 << "\n");
956 llvm_unreachable("Cannot substitute physical registers");
957 } else {
958 DEBUG(dbgs() << "Replacing register (region): "
959 << PrintReg(Register, MRI->getTargetRegisterInfo())
960 << " with "
961 << PrintReg(NewRegister, MRI->getTargetRegisterInfo())
962 << "\n");
963 O.setReg(NewRegister);
964 }
965 }
966 }
967 }
968
969 void LinearizedRegion::replaceRegisterInsideRegion(unsigned Register,
970 unsigned NewRegister,
971 bool IncludeLoopPHIs,
972 MachineRegisterInfo *MRI) {
973 replaceRegister(Register, NewRegister, MRI, true, false, IncludeLoopPHIs);
974 }
975
976 void LinearizedRegion::replaceRegisterOutsideRegion(unsigned Register,
977 unsigned NewRegister,
978 bool IncludeLoopPHIs,
979 MachineRegisterInfo *MRI) {
980 replaceRegister(Register, NewRegister, MRI, false, true, IncludeLoopPHIs);
981 }
982
983 DenseSet *LinearizedRegion::getLiveOuts() { return &LiveOuts; }
984
985 void LinearizedRegion::setEntry(MachineBasicBlock *NewEntry) {
986 Entry = NewEntry;
987 }
988
989 MachineBasicBlock *LinearizedRegion::getEntry() { return Entry; }
990
991 void LinearizedRegion::setExit(MachineBasicBlock *NewExit) { Exit = NewExit; }
992
993 MachineBasicBlock *LinearizedRegion::getExit() { return Exit; }
994
995 void LinearizedRegion::addMBB(MachineBasicBlock *MBB) { MBBs.insert(MBB); }
996
997 void LinearizedRegion::addMBBs(LinearizedRegion *InnerRegion) {
998 for (const auto &MBB : InnerRegion->MBBs) {
999 addMBB(MBB);
1000 }
1001 }
1002
1003 bool LinearizedRegion::contains(MachineBasicBlock *MBB) {
1004 return MBBs.count(MBB) == 1;
1005 }
1006
1007 bool LinearizedRegion::isLiveOut(unsigned Reg) {
1008 return LiveOuts.count(Reg) == 1;
1009 }
1010
1011 bool LinearizedRegion::hasNoDef(unsigned Reg, MachineRegisterInfo *MRI) {
1012 return MRI->def_begin(Reg) == MRI->def_end();
1013 }
1014
1015 bool LinearizedRegion::isDefinedInRegion(unsigned Reg,
1016 MachineRegisterInfo *MRI) {
1017 bool NoDef = hasNoDef(Reg, MRI);
1018 if (NoDef) {
1019 return false;
1020 }
1021
1022 if (!MRI->hasOneDef(Reg)) {
1023 DEBUG(dbgs() << "Register " << PrintReg(Reg, MRI->getTargetRegisterInfo())
1024 << " has multiple defs\n");
1025 }
1026
1027 assert(MRI->hasOneDef(Reg) && "Register has multiple definitions");
1028 MachineOperand *Def = &(*(MRI->def_begin(Reg)));
1029 return contains(Def->getParent()->getParent());
1030 }
1031
1032 // After the code has been structurized, what was flagged as kills
1033 // before are no longer register kills.
1034 void LinearizedRegion::removeFalseRegisterKills(MachineRegisterInfo *MRI) {
1035 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
1036 for (auto MBBI : MBBs) {
1037 MachineBasicBlock *MBB = MBBI;
1038 for (auto &II : *MBB) {
1039 for (auto &RI : II.uses()) {
1040 if (RI.isReg()) {
1041 unsigned Reg = RI.getReg();
1042 if (TRI->isVirtualRegister(Reg)) {
1043 if (hasNoDef(Reg, MRI))
1044 continue;
1045 if (!MRI->hasOneDef(Reg)) {
1046 DEBUG(this->getEntry()->getParent()->dump());
1047 DEBUG(dbgs() << PrintReg(Reg, TRI) << "\n");
1048 }
1049
1050 if (MRI->def_begin(Reg) == MRI->def_end()) {
1051 DEBUG(dbgs() << "Register "
1052 << PrintReg(Reg, MRI->getTargetRegisterInfo())
1053 << " has NO defs\n");
1054 } else if (!MRI->hasOneDef(Reg)) {
1055 DEBUG(dbgs() << "Register "
1056 << PrintReg(Reg, MRI->getTargetRegisterInfo())
1057 << " has multiple defs\n");
1058 }
1059
1060 assert(MRI->hasOneDef(Reg) && "Register has multiple definitions");
1061 MachineOperand *Def = &(*(MRI->def_begin(Reg)));
1062 MachineOperand *UseOperand = &(RI);
1063 bool UseIsOutsideDefMBB = Def->getParent()->getParent() != MBB;
1064 if (UseIsOutsideDefMBB && UseOperand->isKill()) {
1065 DEBUG(dbgs() << "Removing kill flag on register: "
1066 << PrintReg(Reg, TRI) << "\n");
1067 UseOperand->setIsKill(false);
1068 }
1069 }
1070 }
1071 }
1072 }
1073 }
1074 }
1075
1076 void LinearizedRegion::initLiveOut(RegionMRT *Region,
1077 const MachineRegisterInfo *MRI,
1078 const TargetRegisterInfo *TRI,
1079 PHILinearize &PHIInfo) {
1080 storeLiveOuts(Region, MRI, TRI, PHIInfo);
1081 }
1082
1083 LinearizedRegion::LinearizedRegion(MachineBasicBlock *MBB,
1084 const MachineRegisterInfo *MRI,
1085 const TargetRegisterInfo *TRI,
1086 PHILinearize &PHIInfo) {
1087 setEntry(MBB);
1088 setExit(MBB);
1089 storeLiveOuts(MBB, MRI, TRI, PHIInfo);
1090 MBBs.insert(MBB);
1091 Parent = nullptr;
1092 }
1093
1094 LinearizedRegion::LinearizedRegion(RegionMRT *Region,
1095 const MachineRegisterInfo *MRI,
1096 const TargetRegisterInfo *TRI,
1097 PHILinearize &PHIInfo) {
1098 setEntry(Region->getEntry());
1099 // We don't have a single exit block that is part of the region
1100 // at this point. When the transform is performed this block
1101 // will be created.
1102 setExit(nullptr);
1103 storeLiveOuts(Region, MRI, TRI, PHIInfo);
1104 Parent = nullptr;
1105 }
1106
1107 LinearizedRegion::LinearizedRegion() {
1108 setEntry(nullptr);
1109 setExit(nullptr);
1110 Parent = nullptr;
1111 }
1112
1113 LinearizedRegion::~LinearizedRegion() {}
1114
1115 class AMDGPUMachineCFGStructurizer : public MachineFunctionPass {
1116 private:
1117 const MachineRegionInfo *Regions;
1118 const SIInstrInfo *TII;
1119 const TargetRegisterInfo *TRI;
1120 MachineRegisterInfo *MRI;
1121 unsigned BBSelectRegister;
1122 PHILinearize PHIInfo;
1123 DenseMap FallthroughMap;
1124
1125 void getPHIRegionIndices(RegionMRT *Region, MachineInstr &PHI,
1126 SmallVector &RegionIndices);
1127 void getPHIRegionIndices(LinearizedRegion *Region, MachineInstr &PHI,
1128 SmallVector &RegionIndices);
1129 void getPHINonRegionIndices(LinearizedRegion *Region, MachineInstr &PHI,
1130 SmallVector &PHINonRegionIndices);
1131
1132 void storePHILinearizationInfoDest(
1133 unsigned LDestReg, MachineInstr &PHI,
1134 SmallVector *RegionIndices = nullptr);
1135
1136 unsigned storePHILinearizationInfo(MachineInstr &PHI,
1137 SmallVector *RegionIndices);
1138
1139 void extractKilledPHIs(MachineBasicBlock *MBB);
1140 void extractKilledPHIs(LinearizedRegion *LRegion);
1141
1142 bool shrinkPHI(MachineInstr &PHI, SmallVector &PHIIndices,
1143 unsigned *ReplaceReg);
1144
1145 bool shrinkPHI(MachineInstr &PHI, unsigned CombinedSourceReg,
1146 MachineBasicBlock *SourceMBB,
1147 SmallVector &PHIIndices, unsigned *ReplaceReg);
1148
1149 void replacePHI(MachineInstr &PHI, unsigned CombinedSourceReg,
1150 MachineBasicBlock *LastMerge,
1151 SmallVector &PHIRegionIndices);
1152 void replaceEntryPHI(MachineInstr &PHI, unsigned CombinedSourceReg,
1153 MachineBasicBlock *IfMBB,
1154 SmallVector &PHIRegionIndices);
1155 void replaceLiveOutRegs(MachineInstr &PHI,
1156 SmallVector &PHIRegionIndices,
1157 unsigned CombinedSourceReg,
1158 LinearizedRegion *LRegion);
1159 void rewriteRegionExitPHI(RegionMRT *Region, MachineBasicBlock *LastMerge,
1160 MachineInstr &PHI, LinearizedRegion *LRegion);
1161
1162 void rewriteRegionExitPHIs(RegionMRT *Region, MachineBasicBlock *LastMerge,
1163 LinearizedRegion *LRegion);
1164 void rewriteRegionEntryPHI(LinearizedRegion *Region, MachineBasicBlock *IfMBB,
1165 MachineInstr &PHI);
1166 void rewriteRegionEntryPHIs(LinearizedRegion *Region,
1167 MachineBasicBlock *IfMBB);
1168
1169 bool regionIsSimpleIf(RegionMRT *Region);
1170
1171 void transformSimpleIfRegion(RegionMRT *Region);
1172
1173 bool regionIsSimpleLoop(RegionMRT *Region);
1174
1175 void transformSimpleLoopRegion(RegionMRT *Region);
1176
1177 void eliminateDeadBranchOperands(MachineBasicBlock::instr_iterator &II);
1178
1179 void insertUnconditionalBranch(MachineBasicBlock *MBB,
1180 MachineBasicBlock *Dest,
1181 const DebugLoc &DL = DebugLoc());
1182
1183 MachineBasicBlock *createLinearizedExitBlock(RegionMRT *Region);
1184
1185 void replaceRegisterOutsideMBB(MachineBasicBlock *MBB, unsigned Register,
1186 unsigned NewRegister);
1187
1188 void insertMergePHI(MachineBasicBlock *IfBB, MachineBasicBlock *CodeBB,
1189 MachineBasicBlock *MergeBB, unsigned DestRegister,
1190 unsigned IfSourceRegister, unsigned CodeSourceRegister,
1191 bool IsUndefIfSource = false);
1192
1193 MachineBasicBlock *createIfBlock(MachineBasicBlock *MergeBB,
1194 MachineBasicBlock *CodeBBStart,
1195 MachineBasicBlock *CodeBBEnd,
1196 MachineBasicBlock *SelectBB, unsigned IfReg,
1197 bool InheritPreds);
1198
1199 void prunePHIInfo(MachineBasicBlock *MBB);
1200 void createEntryPHI(LinearizedRegion *CurrentRegion, unsigned DestReg);
1201
1202 void createEntryPHIs(LinearizedRegion *CurrentRegion);
1203 void resolvePHIInfos(MachineBasicBlock *FunctionEntry);
1204
1205 void replaceRegisterWith(unsigned Register, unsigned NewRegister);
1206
1207 MachineBasicBlock *createIfRegion(MachineBasicBlock *MergeBB,
1208 MachineBasicBlock *CodeBB,
1209 LinearizedRegion *LRegion,
1210 unsigned BBSelectRegIn,
1211 unsigned BBSelectRegOut);
1212
1213 MachineBasicBlock *
1214 createIfRegion(MachineBasicBlock *MergeMBB, LinearizedRegion *InnerRegion,
1215 LinearizedRegion *CurrentRegion, MachineBasicBlock *SelectBB,
1216 unsigned BBSelectRegIn, unsigned BBSelectRegOut);
1217 void ensureCondIsNotKilled(SmallVector Cond);
1218
1219 void rewriteCodeBBTerminator(MachineBasicBlock *CodeBB,
1220 MachineBasicBlock *MergeBB,
1221 unsigned BBSelectReg);
1222
1223 MachineInstr *getDefInstr(unsigned Reg);
1224 void insertChainedPHI(MachineBasicBlock *IfBB, MachineBasicBlock *CodeBB,
1225 MachineBasicBlock *MergeBB,
1226 LinearizedRegion *InnerRegion, unsigned DestReg,
1227 unsigned SourceReg);
1228 bool containsDef(MachineBasicBlock *MBB, LinearizedRegion *InnerRegion,
1229 unsigned Register);
1230 void rewriteLiveOutRegs(MachineBasicBlock *IfBB, MachineBasicBlock *CodeBB,
1231 MachineBasicBlock *MergeBB,
1232 LinearizedRegion *InnerRegion,
1233 LinearizedRegion *LRegion);
1234
1235 void splitLoopPHI(MachineInstr &PHI, MachineBasicBlock *Entry,
1236 MachineBasicBlock *EntrySucc, LinearizedRegion *LRegion);
1237 void splitLoopPHIs(MachineBasicBlock *Entry, MachineBasicBlock *EntrySucc,
1238 LinearizedRegion *LRegion);
1239
1240 MachineBasicBlock *splitExit(LinearizedRegion *LRegion);
1241
1242 MachineBasicBlock *splitEntry(LinearizedRegion *LRegion);
1243
1244 LinearizedRegion *initLinearizedRegion(RegionMRT *Region);
1245
1246 bool structurizeComplexRegion(RegionMRT *Region);
1247
1248 bool structurizeRegion(RegionMRT *Region);
1249
1250 bool structurizeRegions(RegionMRT *Region, bool isTopRegion);
1251
1252 public:
1253 static char ID;
1254
1255 void getAnalysisUsage(AnalysisUsage &AU) const override {
1256 AU.addRequired();
1257 MachineFunctionPass::getAnalysisUsage(AU);
1258 }
1259
1260 AMDGPUMachineCFGStructurizer() : MachineFunctionPass(ID) {
1261 initializeAMDGPUMachineCFGStructurizerPass(*PassRegistry::getPassRegistry());
1262 }
1263
1264 void initFallthroughMap(MachineFunction &MF);
1265
1266 void createLinearizedRegion(RegionMRT *Region, unsigned SelectOut);
1267
1268 unsigned initializeSelectRegisters(MRT *MRT, unsigned ExistingExitReg,
1269 MachineRegisterInfo *MRI,
1270 const SIInstrInfo *TII);
1271
1272 RegionMRT *RMRT;
1273 void setRegionMRT(RegionMRT *RegionTree) { RMRT = RegionTree; }
1274
1275 RegionMRT *getRegionMRT() { return RMRT; }
1276
1277 bool runOnMachineFunction(MachineFunction &MF) override;
1278 };
1279 }
1280
1281 char AMDGPUMachineCFGStructurizer::ID = 0;
1282
1283 bool AMDGPUMachineCFGStructurizer::regionIsSimpleIf(RegionMRT *Region) {
1284 MachineBasicBlock *Entry = Region->getEntry();
1285 MachineBasicBlock *Succ = Region->getSucc();
1286 bool FoundBypass = false;
1287 bool FoundIf = false;
1288
1289 if (Entry->succ_size() != 2) {
1290 return false;
1291 }
1292
1293 for (MachineBasicBlock::const_succ_iterator SI = Entry->succ_begin(),
1294 E = Entry->succ_end();
1295 SI != E; ++SI) {
1296 MachineBasicBlock *Current = *SI;
1297
1298 if (Current == Succ) {
1299 FoundBypass = true;
1300 } else if ((Current->succ_size() == 1) &&
1301 *(Current->succ_begin()) == Succ) {
1302 FoundIf = true;
1303 }
1304 }
1305
1306 return FoundIf && FoundBypass;
1307 }
1308
1309 void AMDGPUMachineCFGStructurizer::transformSimpleIfRegion(RegionMRT *Region) {
1310 MachineBasicBlock *Entry = Region->getEntry();
1311 MachineBasicBlock *Exit = Region->getExit();
1312 TII->convertNonUniformIfRegion(Entry, Exit);
1313 }
1314
1315 bool AMDGPUMachineCFGStructurizer::regionIsSimpleLoop(RegionMRT *Region) {
1316 MachineBasicBlock *Entry = Region->getEntry();
1317
1318 if (Entry->succ_size() != 1) {
1319 return false;
1320 }
1321
1322 int NumRegionExitEdges = 0;
1323 MachineBasicBlock *BackBlock = nullptr;
1324 for (MachineBasicBlock::const_pred_iterator PI = Entry->succ_begin(),
1325 PE = Entry->succ_end();
1326 PI != PE; ++PI) {
1327 MachineBasicBlock *CurrentSP = *PI;
1328 if (Region->contains(CurrentSP)) {
1329 BackBlock = CurrentSP;
1330 }
1331 }
1332
1333 bool HasBackedge = false;
1334
1335 for (MachineBasicBlock::const_succ_iterator SI = Entry->succ_begin(),
1336 SE = Entry->succ_end();
1337 SI != SE; ++SI) {
1338 MachineBasicBlock *CurrentBS = *SI;
1339 if (CurrentBS == Entry) {
1340 HasBackedge = true;
1341 }
1342 }
1343
1344 return NumRegionExitEdges == 1 && BackBlock->succ_size() == 2 && HasBackedge;
1345 }
1346
1347 void AMDGPUMachineCFGStructurizer::transformSimpleLoopRegion(RegionMRT *Region) {
1348 MachineBasicBlock *Entry = Region->getEntry();
1349 MachineBasicBlock *BackBlock = nullptr;
1350 for (MachineBasicBlock::const_pred_iterator PI = Entry->succ_begin(),
1351 PE = Entry->succ_end();
1352 PI != PE; ++PI) {
1353 MachineBasicBlock *CurrentSP = *PI;
1354 if (Region->contains(CurrentSP)) {
1355 BackBlock = CurrentSP;
1356 }
1357 }
1358
1359 TII->convertNonUniformLoopRegion(Entry, BackBlock);
1360 }
1361
1362 static void fixMBBTerminator(MachineBasicBlock *MBB) {
1363
1364 if (MBB->succ_size() == 1) {
1365 auto *Succ = *(MBB->succ_begin());
1366 for (auto &TI : MBB->terminators()) {
1367 for (auto &UI : TI.uses()) {
1368 if (UI.isMBB() && UI.getMBB() != Succ) {
1369 UI.setMBB(Succ);
1370 }
1371 }
1372 }
1373 }
1374 }
1375
1376 static void fixRegionTerminator(RegionMRT *Region) {
1377 MachineBasicBlock *InternalSucc = nullptr;
1378 MachineBasicBlock *ExternalSucc = nullptr;
1379 LinearizedRegion *LRegion = Region->getLinearizedRegion();
1380 auto Exit = LRegion->getExit();
1381
1382 SmallPtrSet Successors;
1383 for (MachineBasicBlock::const_succ_iterator SI = Exit->succ_begin(),
1384 SE = Exit->succ_end();
1385 SI != SE; ++SI) {
1386 MachineBasicBlock *Succ = *SI;
1387 if (LRegion->contains(Succ)) {
1388 // Do not allow re-assign
1389 assert(InternalSucc == nullptr);
1390 InternalSucc = Succ;
1391 } else {
1392 // Do not allow re-assign
1393 assert(ExternalSucc == nullptr);
1394 ExternalSucc = Succ;
1395 }
1396 }
1397
1398 for (auto &TI : Exit->terminators()) {
1399 for (auto &UI : TI.uses()) {
1400 if (UI.isMBB()) {
1401 auto Target = UI.getMBB();
1402 if (Target != InternalSucc && Target != ExternalSucc) {
1403 UI.setMBB(ExternalSucc);
1404 }
1405 }
1406 }
1407 }
1408 }
1409
1410 // If a region region is just a sequence of regions (and the exit
1411 // block in the case of the top level region), we can simply skip
1412 // linearizing it, because it is already linear
1413 bool regionIsSequence(RegionMRT *Region) {
1414 auto Children = Region->getChildren();
1415 for (auto CI : *Children) {
1416 if (!CI->isRegion()) {
1417 if (CI->getMBBMRT()->getMBB()->succ_size() > 1) {
1418 return false;
1419 }
1420 }
1421 }
1422 return true;
1423 }
1424
1425 void fixupRegionExits(RegionMRT *Region) {
1426 auto Children = Region->getChildren();
1427 for (auto CI : *Children) {
1428 if (!CI->isRegion()) {
1429 fixMBBTerminator(CI->getMBBMRT()->getMBB());
1430 } else {
1431 fixRegionTerminator(CI->getRegionMRT());
1432 }
1433 }
1434 }
1435
1436 void AMDGPUMachineCFGStructurizer::getPHIRegionIndices(
1437 RegionMRT *Region, MachineInstr &PHI,
1438 SmallVector &PHIRegionIndices) {
1439 unsigned NumInputs = getPHINumInputs(PHI);
1440 for (unsigned i = 0; i < NumInputs; ++i) {
1441 MachineBasicBlock *Pred = getPHIPred(PHI, i);
1442 if (Region->contains(Pred)) {
1443 PHIRegionIndices.push_back(i);
1444 }
1445 }
1446 }
1447
1448 void AMDGPUMachineCFGStructurizer::getPHIRegionIndices(
1449 LinearizedRegion *Region, MachineInstr &PHI,
1450 SmallVector &PHIRegionIndices) {
1451 unsigned NumInputs = getPHINumInputs(PHI);
1452 for (unsigned i = 0; i < NumInputs; ++i) {
1453 MachineBasicBlock *Pred = getPHIPred(PHI, i);
1454 if (Region->contains(Pred)) {
1455 PHIRegionIndices.push_back(i);
1456 }
1457 }
1458 }
1459
1460 void AMDGPUMachineCFGStructurizer::getPHINonRegionIndices(
1461 LinearizedRegion *Region, MachineInstr &PHI,
1462 SmallVector &PHINonRegionIndices) {
1463 unsigned NumInputs = getPHINumInputs(PHI);
1464 for (unsigned i = 0; i < NumInputs; ++i) {
1465 MachineBasicBlock *Pred = getPHIPred(PHI, i);
1466 if (!Region->contains(Pred)) {
1467 PHINonRegionIndices.push_back(i);
1468 }
1469 }
1470 }
1471
1472 void AMDGPUMachineCFGStructurizer::storePHILinearizationInfoDest(
1473 unsigned LDestReg, MachineInstr &PHI,
1474 SmallVector *RegionIndices) {
1475 if (RegionIndices) {
1476 for (auto i : *RegionIndices) {
1477 PHIInfo.addSource(LDestReg, getPHISourceReg(PHI, i), getPHIPred(PHI, i));
1478 }
1479 } else {
1480 unsigned NumInputs = getPHINumInputs(PHI);
1481 for (unsigned i = 0; i < NumInputs; ++i) {
1482 PHIInfo.addSource(LDestReg, getPHISourceReg(PHI, i), getPHIPred(PHI, i));
1483 }
1484 }
1485 }
1486
1487 unsigned AMDGPUMachineCFGStructurizer::storePHILinearizationInfo(
1488 MachineInstr &PHI, SmallVector *RegionIndices) {
1489 unsigned DestReg = getPHIDestReg(PHI);
1490 unsigned LinearizeDestReg =
1491 MRI->createVirtualRegister(MRI->getRegClass(DestReg));
1492 PHIInfo.addDest(LinearizeDestReg, PHI.getDebugLoc());
1493 storePHILinearizationInfoDest(LinearizeDestReg, PHI, RegionIndices);
1494 return LinearizeDestReg;
1495 }
1496
1497 void AMDGPUMachineCFGStructurizer::extractKilledPHIs(MachineBasicBlock *MBB) {
1498 // We need to create a new chain for the killed phi, but there is no
1499 // need to do the renaming outside or inside the block.
1500 SmallPtrSet PHIs;
1501 for (MachineBasicBlock::instr_iterator I = MBB->instr_begin(),
1502 E = MBB->instr_end();
1503 I != E; ++I) {
1504 MachineInstr &Instr = *I;
1505 if (Instr.isPHI()) {
1506 unsigned PHIDestReg = getPHIDestReg(Instr);
1507 DEBUG(dbgs() << "Extractking killed phi:\n");
1508 DEBUG(Instr.dump());
1509 PHIs.insert(&Instr);
1510 PHIInfo.addDest(PHIDestReg, Instr.getDebugLoc());
1511 storePHILinearizationInfoDest(PHIDestReg, Instr);
1512 }
1513 }
1514
1515 for (auto PI : PHIs) {
1516 PI->eraseFromParent();
1517 }
1518 }
1519
1520 void AMDGPUMachineCFGStructurizer::extractKilledPHIs(LinearizedRegion *LRegion) {
1521 // PHIs can only exist in the entry block.
1522 extractKilledPHIs(LRegion->getEntry());
1523 }
1524
1525 static bool isPHIRegionIndex(SmallVector PHIRegionIndices,
1526 unsigned Index) {
1527 for (auto i : PHIRegionIndices) {
1528 if (i == Index)
1529 return true;
1530 }
1531 return false;
1532 }
1533
1534 bool AMDGPUMachineCFGStructurizer::shrinkPHI(MachineInstr &PHI,
1535 SmallVector &PHIIndices,
1536 unsigned *ReplaceReg) {
1537 return shrinkPHI(PHI, 0, nullptr, PHIIndices, ReplaceReg);
1538 }
1539
1540 bool AMDGPUMachineCFGStructurizer::shrinkPHI(MachineInstr &PHI,
1541 unsigned CombinedSourceReg,
1542 MachineBasicBlock *SourceMBB,
1543 SmallVector &PHIIndices,
1544 unsigned *ReplaceReg) {
1545 DEBUG(dbgs() << "Shrink PHI: ");
1546 DEBUG(PHI.dump());
1547 DEBUG(dbgs() << " to " << PrintReg(getPHIDestReg(PHI), TRI)
1548 << " = PHI(");
1549
1550 bool Replaced = false;
1551 unsigned NumInputs = getPHINumInputs(PHI);
1552 int SingleExternalEntryIndex = -1;
1553 for (unsigned i = 0; i < NumInputs; ++i) {
1554 if (!isPHIRegionIndex(PHIIndices, i)) {
1555 if (SingleExternalEntryIndex == -1) {
1556 // Single entry
1557 SingleExternalEntryIndex = i;
1558 } else {
1559 // Multiple entries
1560 SingleExternalEntryIndex = -2;
1561 }
1562 }
1563 }
1564
1565 if (SingleExternalEntryIndex > -1) {
1566 *ReplaceReg = getPHISourceReg(PHI, SingleExternalEntryIndex);
1567 // We should not rewrite the code, we should only pick up the single value
1568 // that represents the shrunk PHI.
1569 Replaced = true;
1570 } else {
1571 MachineBasicBlock *MBB = PHI.getParent();
1572 MachineInstrBuilder MIB =
1573 BuildMI(*MBB, PHI, PHI.getDebugLoc(), TII->get(TargetOpcode::PHI),
1574 getPHIDestReg(PHI));
1575 if (SourceMBB) {
1576 MIB.addReg(CombinedSourceReg);
1577 MIB.addMBB(SourceMBB);
1578 DEBUG(dbgs() << PrintReg(CombinedSourceReg, TRI) << ", BB#"
1579 << SourceMBB->getNumber());
1580 }
1581
1582 for (unsigned i = 0; i < NumInputs; ++i) {
1583 if (isPHIRegionIndex(PHIIndices, i)) {
1584 continue;
1585 }
1586 unsigned SourceReg = getPHISourceReg(PHI, i);
1587 MachineBasicBlock *SourcePred = getPHIPred(PHI, i);
1588 MIB.addReg(SourceReg);
1589 MIB.addMBB(SourcePred);
1590 DEBUG(dbgs() << PrintReg(SourceReg, TRI) << ", BB#"
1591 << SourcePred->getNumber());
1592 }
1593 DEBUG(dbgs() << ")\n");
1594 }
1595 PHI.eraseFromParent();
1596 return Replaced;
1597 }
1598
1599 void AMDGPUMachineCFGStructurizer::replacePHI(
1600 MachineInstr &PHI, unsigned CombinedSourceReg, MachineBasicBlock *LastMerge,
1601 SmallVector &PHIRegionIndices) {
1602 DEBUG(dbgs() << "Replace PHI: ");
1603 DEBUG(PHI.dump());
1604 DEBUG(dbgs() << " with " << PrintReg(getPHIDestReg(PHI), TRI)
1605 << " = PHI(");
1606
1607 bool HasExternalEdge = false;
1608 unsigned NumInputs = getPHINumInputs(PHI);
1609 for (unsigned i = 0; i < NumInputs; ++i) {
1610 if (!isPHIRegionIndex(PHIRegionIndices, i)) {
1611 HasExternalEdge = true;
1612 }
1613 }
1614
1615 if (HasExternalEdge) {
1616 MachineBasicBlock *MBB = PHI.getParent();
1617 MachineInstrBuilder MIB =
1618 BuildMI(*MBB, PHI, PHI.getDebugLoc(), TII->get(TargetOpcode::PHI),
1619 getPHIDestReg(PHI));
1620 MIB.addReg(CombinedSourceReg);
1621 MIB.addMBB(LastMerge);
1622 DEBUG(dbgs() << PrintReg(CombinedSourceReg, TRI) << ", BB#"
1623 << LastMerge->getNumber());
1624 for (unsigned i = 0; i < NumInputs; ++i) {
1625 if (isPHIRegionIndex(PHIRegionIndices, i)) {
1626 continue;
1627 }
1628 unsigned SourceReg = getPHISourceReg(PHI, i);
1629 MachineBasicBlock *SourcePred = getPHIPred(PHI, i);
1630 MIB.addReg(SourceReg);
1631 MIB.addMBB(SourcePred);
1632 DEBUG(dbgs() << PrintReg(SourceReg, TRI) << ", BB#"
1633 << SourcePred->getNumber());
1634 }
1635 DEBUG(dbgs() << ")\n");
1636 } else {
1637 replaceRegisterWith(getPHIDestReg(PHI), CombinedSourceReg);
1638 }
1639 PHI.eraseFromParent();
1640 }
1641
1642 void AMDGPUMachineCFGStructurizer::replaceEntryPHI(
1643 MachineInstr &PHI, unsigned CombinedSourceReg, MachineBasicBlock *IfMBB,
1644 SmallVector &PHIRegionIndices) {
1645
1646 DEBUG(dbgs() << "Replace entry PHI: ");
1647 DEBUG(PHI.dump());
1648 DEBUG(dbgs() << " with ");
1649
1650 unsigned NumInputs = getPHINumInputs(PHI);
1651 unsigned NumNonRegionInputs = NumInputs;
1652 for (unsigned i = 0; i < NumInputs; ++i) {
1653 if (isPHIRegionIndex(PHIRegionIndices, i)) {
1654 NumNonRegionInputs--;
1655 }
1656 }
1657
1658 if (NumNonRegionInputs == 0) {
1659 auto DestReg = getPHIDestReg(PHI);
1660 replaceRegisterWith(DestReg, CombinedSourceReg);
1661 DEBUG(dbgs() << " register " << PrintReg(CombinedSourceReg, TRI) << "\n");
1662 PHI.eraseFromParent();
1663 } else {
1664 DEBUG(dbgs() << PrintReg(getPHIDestReg(PHI), TRI) << " = PHI(");
1665 MachineBasicBlock *MBB = PHI.getParent();
1666 MachineInstrBuilder MIB =
1667 BuildMI(*MBB, PHI, PHI.getDebugLoc(), TII->get(TargetOpcode::PHI),
1668 getPHIDestReg(PHI));
1669 MIB.addReg(CombinedSourceReg);
1670 MIB.addMBB(IfMBB);
1671 DEBUG(dbgs() << PrintReg(CombinedSourceReg, TRI) << ", BB#"
1672 << IfMBB->getNumber());
1673 unsigned NumInputs = getPHINumInputs(PHI);
1674 for (unsigned i = 0; i < NumInputs; ++i) {
1675 if (isPHIRegionIndex(PHIRegionIndices, i)) {
1676 continue;
1677 }
1678 unsigned SourceReg = getPHISourceReg(PHI, i);
1679 MachineBasicBlock *SourcePred = getPHIPred(PHI, i);
1680 MIB.addReg(SourceReg);
1681 MIB.addMBB(SourcePred);
1682 DEBUG(dbgs() << PrintReg(SourceReg, TRI) << ", BB#"
1683 << SourcePred->getNumber());
1684 }
1685 DEBUG(dbgs() << ")\n");
1686 PHI.eraseFromParent();
1687 }
1688 }
1689
1690 void AMDGPUMachineCFGStructurizer::replaceLiveOutRegs(
1691 MachineInstr &PHI, SmallVector &PHIRegionIndices,
1692 unsigned CombinedSourceReg, LinearizedRegion *LRegion) {
1693 bool WasLiveOut = false;
1694 for (auto PII : PHIRegionIndices) {
1695 unsigned Reg = getPHISourceReg(PHI, PII);
1696 if (LRegion->isLiveOut(Reg)) {
1697 bool IsDead = true;
1698
1699 // Check if register is live out of the basic block
1700 MachineBasicBlock *DefMBB = getDefInstr(Reg)->getParent();
1701 for (auto UI = MRI->use_begin(Reg), E = MRI->use_end(); UI != E; ++UI) {
1702 if ((*UI).getParent()->getParent() != DefMBB) {
1703 IsDead = false;
1704 }
1705 }
1706
1707 DEBUG(dbgs() << "Register " << PrintReg(Reg, TRI) << " is "
1708 << (IsDead ? "dead" : "alive") << " after PHI replace\n");
1709 if (IsDead) {
1710 LRegion->removeLiveOut(Reg);
1711 }
1712 WasLiveOut = true;
1713 }
1714 }
1715
1716 if (WasLiveOut)
1717 LRegion->addLiveOut(CombinedSourceReg);
1718 }
1719
1720 void AMDGPUMachineCFGStructurizer::rewriteRegionExitPHI(RegionMRT *Region,
1721 MachineBasicBlock *LastMerge,
1722 MachineInstr &PHI,
1723 LinearizedRegion *LRegion) {
1724 SmallVector PHIRegionIndices;
1725 getPHIRegionIndices(Region, PHI, PHIRegionIndices);
1726 unsigned LinearizedSourceReg =
1727 storePHILinearizationInfo(PHI, &PHIRegionIndices);
1728
1729 replacePHI(PHI, LinearizedSourceReg, LastMerge, PHIRegionIndices);
1730 replaceLiveOutRegs(PHI, PHIRegionIndices, LinearizedSourceReg, LRegion);
1731 }
1732
1733 void AMDGPUMachineCFGStructurizer::rewriteRegionEntryPHI(LinearizedRegion *Region,
1734 MachineBasicBlock *IfMBB,
1735 MachineInstr &PHI) {
1736 SmallVector PHINonRegionIndices;
1737 getPHINonRegionIndices(Region, PHI, PHINonRegionIndices);
1738 unsigned LinearizedSourceReg =
1739 storePHILinearizationInfo(PHI, &PHINonRegionIndices);
1740 replaceEntryPHI(PHI, LinearizedSourceReg, IfMBB, PHINonRegionIndices);
1741 }
1742
1743 static void collectPHIs(MachineBasicBlock *MBB,
1744 SmallVector &PHIs) {
1745 for (auto &BBI : *MBB) {
1746 if (BBI.isPHI()) {
1747 PHIs.push_back(&BBI);
1748 }
1749 }
1750 }
1751
1752 void AMDGPUMachineCFGStructurizer::rewriteRegionExitPHIs(RegionMRT *Region,
1753 MachineBasicBlock *LastMerge,
1754 LinearizedRegion *LRegion) {
1755 SmallVector PHIs;
1756 auto Exit = Region->getSucc();
1757 if (Exit == nullptr)
1758 return;
1759
1760 collectPHIs(Exit, PHIs);
1761
1762 for (auto PHII : PHIs) {
1763 rewriteRegionExitPHI(Region, LastMerge, *PHII, LRegion);
1764 }
1765 }
1766
1767 void AMDGPUMachineCFGStructurizer::rewriteRegionEntryPHIs(LinearizedRegion *Region,
1768 MachineBasicBlock *IfMBB) {
1769 SmallVector PHIs;
1770 auto Entry = Region->getEntry();
1771
1772 collectPHIs(Entry, PHIs);
1773
1774 for (auto PHII : PHIs) {
1775 rewriteRegionEntryPHI(Region, IfMBB, *PHII);
1776 }
1777 }
1778
1779 void AMDGPUMachineCFGStructurizer::insertUnconditionalBranch(MachineBasicBlock *MBB,
1780 MachineBasicBlock *Dest,
1781 const DebugLoc &DL) {
1782 DEBUG(dbgs() << "Inserting unconditional branch: " << MBB->getNumber()
1783 << " -> " << Dest->getNumber() << "\n");
1784 MachineBasicBlock::instr_iterator Terminator = MBB->getFirstInstrTerminator();
1785 bool HasTerminator = Terminator != MBB->instr_end();
1786 if (HasTerminator) {
1787 TII->ReplaceTailWithBranchTo(Terminator, Dest);
1788 }
1789 if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(Dest)) {
1790 TII->insertUnconditionalBranch(*MBB, Dest, DL);
1791 }
1792 }
1793
1794 static MachineBasicBlock *getSingleExitNode(MachineFunction &MF) {
1795 MachineBasicBlock *result = nullptr;
1796 for (auto &MFI : MF) {
1797 if (MFI.succ_size() == 0) {
1798 if (result == nullptr) {
1799 result = &MFI;
1800 } else {
1801 return nullptr;
1802 }
1803 }
1804 }
1805
1806 return result;
1807 }
1808
1809 static bool hasOneExitNode(MachineFunction &MF) {
1810 return getSingleExitNode(MF) != nullptr;
1811 }
1812
1813 MachineBasicBlock *
1814 AMDGPUMachineCFGStructurizer::createLinearizedExitBlock(RegionMRT *Region) {
1815 auto Exit = Region->getSucc();
1816
1817 // If the exit is the end of the function, we just use the existing
1818 MachineFunction *MF = Region->getEntry()->getParent();
1819 if (Exit == nullptr && hasOneExitNode(*MF)) {
1820 return &(*(--(Region->getEntry()->getParent()->end())));
1821 }
1822
1823 MachineBasicBlock *LastMerge = MF->CreateMachineBasicBlock();
1824 if (Exit == nullptr) {
1825 MachineFunction::iterator ExitIter = MF->end();
1826 MF->insert(ExitIter, LastMerge);
1827 } else {
1828 MachineFunction::iterator ExitIter = Exit->getIterator();
1829 MF->insert(ExitIter, LastMerge);
1830 LastMerge->addSuccessor(Exit);
1831 insertUnconditionalBranch(LastMerge, Exit);
1832 DEBUG(dbgs() << "Created exit block: " << LastMerge->getNumber() << "\n");
1833 }
1834 return LastMerge;
1835 }
1836
1837 void AMDGPUMachineCFGStructurizer::replaceRegisterOutsideMBB(MachineBasicBlock *MBB,
1838 unsigned Register,
1839 unsigned NewRegister) {
1840 assert(Register != NewRegister && "Cannot replace a reg with itself");
1841
1842 for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(Register),
1843 E = MRI->reg_end();
1844 I != E;) {
1845 MachineOperand &O = *I;
1846 ++I;
1847 if (O.getParent()->getParent() != MBB) {
1848 if (TargetRegisterInfo::isPhysicalRegister(NewRegister)) {
1849 llvm_unreachable("Cannot substitute physical registers");
1850 // We don't handle physical registers, but if we need to in the future
1851 // This is how we do it: O.substPhysReg(NewRegister, *TRI);
1852 } else {
1853 O.setReg(NewRegister);
1854 }
1855 }
1856 }
1857 }
1858
1859 void AMDGPUMachineCFGStructurizer::insertMergePHI(MachineBasicBlock *IfBB,
1860 MachineBasicBlock *CodeBB,
1861 MachineBasicBlock *MergeBB,
1862 unsigned DestRegister,
1863 unsigned IfSourceRegister,
1864 unsigned CodeSourceRegister,
1865 bool IsUndefIfSource) {
1866 // If this is the function exit block, we don't need a phi.
1867 if (MergeBB->succ_begin() == MergeBB->succ_end()) {
1868 return;
1869 }
1870 DEBUG(dbgs() << "Merge PHI (BB#" << MergeBB->getNumber()
1871 << "): " << PrintReg(DestRegister, TRI) << " = PHI("
1872 << PrintReg(IfSourceRegister, TRI) << ", BB#"
1873 << IfBB->getNumber() << PrintReg(CodeSourceRegister, TRI)
1874 << ", BB#" << CodeBB->getNumber() << ")\n");
1875 const DebugLoc &DL = MergeBB->findDebugLoc(MergeBB->begin());
1876 MachineInstrBuilder MIB = BuildMI(*MergeBB, MergeBB->instr_begin(), DL,
1877 TII->get(TargetOpcode::PHI), DestRegister);
1878 if (IsUndefIfSource && false) {
1879 MIB.addReg(IfSourceRegister, RegState::Undef);
1880 } else {
1881 MIB.addReg(IfSourceRegister);
1882 }
1883 MIB.addMBB(IfBB);
1884 MIB.addReg(CodeSourceRegister);
1885 MIB.addMBB(CodeBB);
1886 }
1887
1888 static void removeExternalCFGSuccessors(MachineBasicBlock *MBB) {
1889 for (MachineBasicBlock::succ_iterator PI = MBB->succ_begin(),
1890 E = MBB->succ_end();
1891 PI != E; ++PI) {
1892 if ((*PI) != MBB) {
1893 (MBB)->removeSuccessor(*PI);
1894 }
1895 }
1896 }
1897
1898 static void removeExternalCFGEdges(MachineBasicBlock *StartMBB,
1899 MachineBasicBlock *EndMBB) {
1900
1901 // We have to check against the StartMBB successor becasuse a
1902 // structurized region with a loop will have the entry block split,
1903 // and the backedge will go to the entry successor.
1904 DenseSet> Succs;
1905 unsigned SuccSize = StartMBB->succ_size();
1906 if (SuccSize > 0) {
1907 MachineBasicBlock *StartMBBSucc = *(StartMBB->succ_begin());
1908 for (MachineBasicBlock::succ_iterator PI = EndMBB->succ_begin(),
1909 E = EndMBB->succ_end();
1910 PI != E; ++PI) {
1911 // Either we have a back-edge to the entry block, or a back-edge to the
1912 // succesor of the entry block since the block may be split.
1913 if ((*PI) != StartMBB &&
1914 !((*PI) == StartMBBSucc && StartMBB != EndMBB && SuccSize == 1)) {
1915 Succs.insert(
1916 std::pair(EndMBB, *PI));
1917 }
1918 }
1919 }
1920
1921 for (MachineBasicBlock::pred_iterator PI = StartMBB->pred_begin(),
1922 E = StartMBB->pred_end();
1923 PI != E; ++PI) {
1924 if ((*PI) != EndMBB) {
1925 Succs.insert(
1926 std::pair(*PI, StartMBB));
1927 }
1928 }
1929
1930 for (auto SI : Succs) {
1931 std::pair Edge = SI;
1932 DEBUG(dbgs() << "Removing edge: BB#" << Edge.first->getNumber() << " -> BB#"
1933 << Edge.second->getNumber() << "\n");
1934 Edge.first->removeSuccessor(Edge.second);
1935 }
1936 }
1937
1938 MachineBasicBlock *AMDGPUMachineCFGStructurizer::createIfBlock(
1939 MachineBasicBlock *MergeBB, MachineBasicBlock *CodeBBStart,
1940 MachineBasicBlock *CodeBBEnd, MachineBasicBlock *SelectBB, unsigned IfReg,
1941 bool InheritPreds) {
1942 MachineFunction *MF = MergeBB->getParent();
1943 MachineBasicBlock *IfBB = MF->CreateMachineBasicBlock();
1944
1945 if (InheritPreds) {
1946 for (MachineBasicBlock::pred_iterator PI = CodeBBStart->pred_begin(),
1947 E = CodeBBStart->pred_end();
1948 PI != E; ++PI) {
1949 if ((*PI) != CodeBBEnd) {
1950 MachineBasicBlock *Pred = (*PI);
1951 Pred->addSuccessor(IfBB);
1952 }
1953 }
1954 }
1955
1956 removeExternalCFGEdges(CodeBBStart, CodeBBEnd);
1957
1958 auto CodeBBStartI = CodeBBStart->getIterator();
1959 auto CodeBBEndI = CodeBBEnd->getIterator();
1960 auto MergeIter = MergeBB->getIterator();
1961 MF->insert(MergeIter, IfBB);
1962 MF->splice(MergeIter, CodeBBStartI, ++CodeBBEndI);
1963 IfBB->addSuccessor(MergeBB);
1964 IfBB->addSuccessor(CodeBBStart);
1965
1966 DEBUG(dbgs() << "Created If block: " << IfBB->getNumber() << "\n");
1967 // Ensure that the MergeBB is a succesor of the CodeEndBB.
1968 if (!CodeBBEnd->isSuccessor(MergeBB))
1969 CodeBBEnd->addSuccessor(MergeBB);
1970
1971 DEBUG(dbgs() << "Moved MBB#" << CodeBBStart->getNumber() << " through MBB#"
1972 << CodeBBEnd->getNumber() << "\n");
1973
1974 // If we have a single predecessor we can find a reasonable debug location
1975 MachineBasicBlock *SinglePred =
1976 CodeBBStart->pred_size() == 1 ? *(CodeBBStart->pred_begin()) : nullptr;
1977 const DebugLoc &DL = SinglePred
1978 ? SinglePred->findDebugLoc(SinglePred->getFirstTerminator())
1979 : DebugLoc();
1980
1981 unsigned Reg =
1982 TII->insertEQ(IfBB, IfBB->begin(), DL, IfReg,
1983 SelectBB->getNumber() /* CodeBBStart->getNumber() */);
1984 if (&(*(IfBB->getParent()->begin())) == IfBB) {
1985 TII->materializeImmediate(*IfBB, IfBB->begin(), DL, IfReg,
1986 CodeBBStart->getNumber());
1987 }
1988 MachineOperand RegOp = MachineOperand::CreateReg(Reg, false, false, true);
1989 ArrayRef Cond(RegOp);
1990 TII->insertBranch(*IfBB, MergeBB, CodeBBStart, Cond, DL);
1991
1992 return IfBB;
1993 }
1994
1995 void AMDGPUMachineCFGStructurizer::ensureCondIsNotKilled(
1996 SmallVector Cond) {
1997 if (Cond.size() != 1)
1998 return;
1999 if (!Cond[0].isReg())
2000 return;
2001
2002 unsigned CondReg = Cond[0].getReg();
2003 for (auto UI = MRI->use_begin(CondReg), E = MRI->use_end(); UI != E; ++UI) {
2004 (*UI).setIsKill(false);
2005 }
2006 }
2007
2008 void AMDGPUMachineCFGStructurizer::rewriteCodeBBTerminator(MachineBasicBlock *CodeBB,
2009 MachineBasicBlock *MergeBB,
2010 unsigned BBSelectReg) {
2011 MachineBasicBlock *TrueBB = nullptr;
2012 MachineBasicBlock *FalseBB = nullptr;
2013 SmallVector Cond;
2014 MachineBasicBlock *FallthroughBB = FallthroughMap[CodeBB];
2015 TII->analyzeBranch(*CodeBB, TrueBB, FalseBB, Cond);
2016
2017 const DebugLoc &DL = CodeBB->findDebugLoc(CodeBB->getFirstTerminator());
2018
2019 if (FalseBB == nullptr && TrueBB == nullptr && FallthroughBB == nullptr) {
2020 // This is an exit block, hence no successors. We will assign the
2021 // bb select register to the entry block.
2022 TII->materializeImmediate(*CodeBB, CodeBB->getFirstTerminator(), DL,
2023 BBSelectReg,
2024 CodeBB->getParent()->begin()->getNumber());
2025 insertUnconditionalBranch(CodeBB, MergeBB, DL);
2026 return;
2027 }
2028
2029 if (FalseBB == nullptr && TrueBB == nullptr) {
2030 TrueBB = FallthroughBB;
2031 } else if (TrueBB != nullptr) {
2032 FalseBB =
2033 (FallthroughBB && (FallthroughBB != TrueBB)) ? FallthroughBB : FalseBB;
2034 }
2035
2036 if ((TrueBB != nullptr && FalseBB == nullptr) || (TrueBB == FalseBB)) {
2037 TII->materializeImmediate(*CodeBB, CodeBB->getFirstTerminator(), DL,
2038 BBSelectReg, TrueBB->getNumber());
2039 } else {
2040 const TargetRegisterClass *RegClass = MRI->getRegClass(BBSelectReg);
2041 unsigned TrueBBReg = MRI->createVirtualRegister(RegClass);
2042 unsigned FalseBBReg = MRI->createVirtualRegister(RegClass);
2043 TII->materializeImmediate(*CodeBB, CodeBB->getFirstTerminator(), DL,
2044 TrueBBReg, TrueBB->getNumber());
2045 TII->materializeImmediate(*CodeBB, CodeBB->getFirstTerminator(), DL,
2046 FalseBBReg, FalseBB->getNumber());
2047 ensureCondIsNotKilled(Cond);
2048 TII->insertVectorSelect(*CodeBB, CodeBB->getFirstTerminator(), DL,
2049 BBSelectReg, Cond, TrueBBReg, FalseBBReg);
2050 }
2051
2052 insertUnconditionalBranch(CodeBB, MergeBB, DL);
2053 }
2054
2055 MachineInstr *AMDGPUMachineCFGStructurizer::getDefInstr(unsigned Reg) {
2056 if (MRI->def_begin(Reg) == MRI->def_end()) {
2057 DEBUG(dbgs() << "Register " << PrintReg(Reg, MRI->getTargetRegisterInfo())
2058 << " has NO defs\n");
2059 } else if (!MRI->hasOneDef(Reg)) {
2060 DEBUG(dbgs() << "Register " << PrintReg(Reg, MRI->getTargetRegisterInfo())
2061 << " has multiple defs\n");
2062 DEBUG(dbgs() << "DEFS BEGIN:\n");
2063 for (auto DI = MRI->def_begin(Reg), DE = MRI->def_end(); DI != DE; ++DI) {
2064 DEBUG(DI->getParent()->dump());
2065 }
2066 DEBUG(dbgs() << "DEFS END\n");
2067 }
2068
2069 assert(MRI->hasOneDef(Reg) && "Register has multiple definitions");
2070 return (*(MRI->def_begin(Reg))).getParent();
2071 }
2072
2073 void AMDGPUMachineCFGStructurizer::insertChainedPHI(MachineBasicBlock *IfBB,
2074 MachineBasicBlock *CodeBB,
2075 MachineBasicBlock *MergeBB,
2076 LinearizedRegion *InnerRegion,
2077 unsigned DestReg,
2078 unsigned SourceReg) {
2079 // In this function we know we are part of a chain already, so we need
2080 // to add the registers to the existing chain, and rename the register
2081 // inside the region.
2082 bool IsSingleBB = InnerRegion->getEntry() == InnerRegion->getExit();
2083 MachineInstr *DefInstr = getDefInstr(SourceReg);
2084 if (DefInstr->isPHI() && DefInstr->getParent() == CodeBB && IsSingleBB) {
2085 // Handle the case where the def is a PHI-def inside a basic
2086 // block, then we only need to do renaming. Special care needs to
2087 // be taken if the PHI-def is part of an existing chain, or if a
2088 // new one needs to be created.
2089 InnerRegion->replaceRegisterInsideRegion(SourceReg, DestReg, true, MRI);
2090
2091 // We collect all PHI Information, and if we are at the region entry,
2092 // all PHIs will be removed, and then re-introduced if needed.
2093 storePHILinearizationInfoDest(DestReg, *DefInstr);
2094 // We have picked up all the information we need now and can remove
2095 // the PHI
2096 PHIInfo.removeSource(DestReg, SourceReg, CodeBB);
2097 DefInstr->eraseFromParent();
2098 } else {
2099 // If this is not a phi-def, or it is a phi-def but from a linearized region
2100 if (IsSingleBB && DefInstr->getParent() == InnerRegion->getEntry()) {
2101 // If this is a single BB and the definition is in this block we
2102 // need to replace any uses outside the region.
2103 InnerRegion->replaceRegisterOutsideRegion(SourceReg, DestReg, false, MRI);
2104 }
2105 const TargetRegisterClass *RegClass = MRI->getRegClass(DestReg);
2106 unsigned NextDestReg = MRI->createVirtualRegister(RegClass);
2107 bool IsLastDef = PHIInfo.getNumSources(DestReg) == 1;
2108 DEBUG(dbgs() << "Insert Chained PHI\n");
2109 insertMergePHI(IfBB, InnerRegion->getExit(), MergeBB, DestReg, NextDestReg,
2110 SourceReg, IsLastDef);
2111
2112 PHIInfo.removeSource(DestReg, SourceReg, CodeBB);
2113 if (IsLastDef) {
2114 const DebugLoc &DL = IfBB->findDebugLoc(IfBB->getFirstTerminator());
2115 TII->materializeImmediate(*IfBB, IfBB->getFirstTerminator(), DL,
2116 NextDestReg, 0);
2117 PHIInfo.deleteDef(DestReg);
2118 } else {
2119 PHIInfo.replaceDef(DestReg, NextDestReg);
2120 }
2121 }
2122 }
2123
2124 bool AMDGPUMachineCFGStructurizer::containsDef(MachineBasicBlock *MBB,
2125 LinearizedRegion *InnerRegion,
2126 unsigned Register) {
2127 return getDefInstr(Register)->getParent() == MBB ||
2128 InnerRegion->contains(getDefInstr(Register)->getParent());
2129 }
2130
2131 void AMDGPUMachineCFGStructurizer::rewriteLiveOutRegs(MachineBasicBlock *IfBB,
2132 MachineBasicBlock *CodeBB,
2133 MachineBasicBlock *MergeBB,
2134 LinearizedRegion *InnerRegion,
2135 LinearizedRegion *LRegion) {
2136 DenseSet *LiveOuts = InnerRegion->getLiveOuts();
2137 SmallVector OldLiveOuts;
2138 bool IsSingleBB = InnerRegion->getEntry() == InnerRegion->getExit();
2139 for (auto OLI : *LiveOuts) {
2140 OldLiveOuts.push_back(OLI);
2141 }
2142
2143 for (auto LI : OldLiveOuts) {
2144 DEBUG(dbgs() << "LiveOut: " << PrintReg(LI, TRI));
2145 if (!containsDef(CodeBB, InnerRegion, LI) ||
2146 (!IsSingleBB && (getDefInstr(LI)->getParent() == LRegion->getExit()))) {
2147 // If the register simly lives through the CodeBB, we don't have
2148 // to rewrite anything since the register is not defined in this
2149 // part of the code.
2150 DEBUG(dbgs() << "- through");
2151 continue;
2152 }
2153 DEBUG(dbgs() << "\n");
2154 unsigned Reg = LI;
2155 if (/*!PHIInfo.isSource(Reg) &&*/ Reg != InnerRegion->getBBSelectRegOut()) {
2156 // If the register is live out, we do want to create a phi,
2157 // unless it is from the Exit block, becasuse in that case there
2158 // is already a PHI, and no need to create a new one.
2159
2160 // If the register is just a live out def and not part of a phi
2161 // chain, we need to create a PHI node to handle the if region,
2162 // and replace all uses outside of the region with the new dest
2163 // register, unless it is the outgoing BB select register. We have
2164 // already creaed phi nodes for these.
2165 const TargetRegisterClass *RegClass = MRI->getRegClass(Reg);
2166 unsigned PHIDestReg = MRI->createVirtualRegister(RegClass);
2167 unsigned IfSourceReg = MRI->createVirtualRegister(RegClass);
2168 // Create initializer, this value is never used, but is needed
2169 // to satisfy SSA.
2170 DEBUG(dbgs() << "Initializer for reg: " << PrintReg(Reg) << "\n");
2171 TII->materializeImmediate(*IfBB, IfBB->getFirstTerminator(), DebugLoc(),
2172 IfSourceReg, 0);
2173
2174 InnerRegion->replaceRegisterOutsideRegion(Reg, PHIDestReg, true, MRI);
2175 DEBUG(dbgs() << "Insert Non-Chained Live out PHI\n");
2176 insertMergePHI(IfBB, InnerRegion->getExit(), MergeBB, PHIDestReg,
2177 IfSourceReg, Reg, true);
2178 }
2179 }
2180
2181 // Handle the chained definitions in PHIInfo, checking if this basic block
2182 // is a source block for a definition.
2183 SmallVector Sources;
2184 if (PHIInfo.findSourcesFromMBB(CodeBB, Sources)) {
2185 DEBUG(dbgs() << "Inserting PHI Live Out from BB#" << CodeBB->getNumber()
2186 << "\n");
2187 for (auto SI : Sources) {
2188 unsigned DestReg;
2189 PHIInfo.findDest(SI, CodeBB, DestReg);
2190 insertChainedPHI(IfBB, CodeBB, MergeBB, InnerRegion, DestReg, SI);
2191 }
2192 DEBUG(dbgs() << "Insertion done.\n");
2193 }
2194
2195 PHIInfo.dump(MRI);
2196 }
2197
2198 void AMDGPUMachineCFGStructurizer::prunePHIInfo(MachineBasicBlock *MBB) {
2199 DEBUG(dbgs() << "Before PHI Prune\n");
2200 DEBUG(PHIInfo.dump(MRI));
2201 SmallVector, 4>
2202 ElimiatedSources;
2203 for (auto DRI = PHIInfo.dests_begin(), DE = PHIInfo.dests_end(); DRI != DE;
2204 ++DRI) {
2205
2206 unsigned DestReg = *DRI;
2207 auto SE = PHIInfo.sources_end(DestReg);
2208
2209 bool MBBContainsPHISource = false;
2210 // Check if there is a PHI source in this MBB
2211 for (auto SRI = PHIInfo.sources_begin(DestReg); SRI != SE; ++SRI) {
2212 unsigned SourceReg = (*SRI).first;
2213 MachineOperand *Def = &(*(MRI->def_begin(SourceReg)));
2214 if (Def->getParent()->getParent() == MBB) {
2215 MBBContainsPHISource = true;
2216 }
2217 }
2218
2219 // If so, all other sources are useless since we know this block
2220 // is always executed when the region is executed.
2221 if (MBBContainsPHISource) {
2222 for (auto SRI = PHIInfo.sources_begin(DestReg); SRI != SE; ++SRI) {
2223 PHILinearize::PHISourceT Source = *SRI;
2224 unsigned SourceReg = Source.first;
2225 MachineBasicBlock *SourceMBB = Source.second;
2226 MachineOperand *Def = &(*(MRI->def_begin(SourceReg)));
2227 if (Def->getParent()->getParent() != MBB) {
2228 ElimiatedSources.push_back(
2229 std::make_tuple(DestReg, SourceReg, SourceMBB));
2230 }
2231 }
2232 }
2233 }
2234
2235 // Remove the PHI sources that are in the given MBB
2236 for (auto &SourceInfo : ElimiatedSources) {
2237 PHIInfo.removeSource(std::get<0>(SourceInfo), std::get<1>(SourceInfo),
2238 std::get<2>(SourceInfo));
2239 }
2240 DEBUG(dbgs() << "After PHI Prune\n");
2241 DEBUG(PHIInfo.dump(MRI));
2242 }
2243
2244 void AMDGPUMachineCFGStructurizer::createEntryPHI(LinearizedRegion *CurrentRegion,
2245 unsigned DestReg) {
2246 MachineBasicBlock *Entry = CurrentRegion->getEntry();
2247 MachineBasicBlock *Exit = CurrentRegion->getExit();
2248 MachineBasicBlock *Pred = *(Entry->pred_begin());
2249
2250 DEBUG(dbgs() << "RegionExit: " << Exit->getNumber()
2251 << " Pred: " << Pred->getNumber() << "\n");
2252
2253 int NumSources = 0;
2254 auto SE = PHIInfo.sources_end(DestReg);
2255
2256 for (auto SRI = PHIInfo.sources_begin(DestReg); SRI != SE; ++SRI) {
2257 NumSources++;
2258 }
2259
2260 if (NumSources == 1) {
2261 auto SRI = PHIInfo.sources_begin(DestReg);
2262 unsigned SourceReg = (*SRI).first;
2263 replaceRegisterWith(DestReg, SourceReg);
2264 } else {
2265 const DebugLoc &DL = Entry->findDebugLoc(Entry->begin());
2266 MachineInstrBuilder MIB = BuildMI(*Entry, Entry->instr_begin(), DL,
2267 TII->get(TargetOpcode::PHI), DestReg);
2268 DEBUG(dbgs() << "Entry PHI " << PrintReg(DestReg, TRI) << " = PHI(");
2269
2270 unsigned CurrentBackedgeReg = 0;
2271
2272 for (auto SRI = PHIInfo.sources_begin(DestReg); SRI != SE; ++SRI) {
2273 unsigned SourceReg = (*SRI).first;
2274
2275 if (CurrentRegion->contains((*SRI).second)) {
2276 if (CurrentBackedgeReg == 0) {
2277 CurrentBackedgeReg = SourceReg;
2278 } else {
2279 MachineInstr *PHIDefInstr = getDefInstr(SourceReg);
2280 MachineBasicBlock *PHIDefMBB = PHIDefInstr->getParent();
2281 const TargetRegisterClass *RegClass =
2282 MRI->getRegClass(CurrentBackedgeReg);
2283 unsigned NewBackedgeReg = MRI->createVirtualRegister(RegClass);
2284 MachineInstrBuilder BackedgePHI =
2285 BuildMI(*PHIDefMBB, PHIDefMBB->instr_begin(), DL,
2286 TII->get(TargetOpcode::PHI), NewBackedgeReg);
2287 BackedgePHI.addReg(CurrentBackedgeReg);
2288 BackedgePHI.addMBB(getPHIPred(*PHIDefInstr, 0));
2289 BackedgePHI.addReg(getPHISourceReg(*PHIDefInstr, 1));
2290 BackedgePHI.addMBB((*SRI).second);
2291 CurrentBackedgeReg = NewBackedgeReg;
2292 DEBUG(dbgs() << "Inserting backedge PHI: "
2293 << PrintReg(NewBackedgeReg, TRI) << " = PHI("
2294 << PrintReg(CurrentBackedgeReg, TRI) << ", BB#"
2295 << getPHIPred(*PHIDefInstr, 0)->getNumber() << ", "
2296 << PrintReg(getPHISourceReg(*PHIDefInstr, 1), TRI)
2297 << ", BB#" << (*SRI).second->getNumber());
2298 }
2299 } else {
2300 MIB.addReg(SourceReg);
2301 MIB.addMBB((*SRI).second);
2302 DEBUG(dbgs() << PrintReg(SourceReg, TRI) << ", BB#"
2303 << (*SRI).second->getNumber() << ", ");
2304 }
2305 }
2306
2307 // Add the final backedge register source to the entry phi
2308 if (CurrentBackedgeReg != 0) {
2309 MIB.addReg(CurrentBackedgeReg);
2310 MIB.addMBB(Exit);
2311 DEBUG(dbgs() << PrintReg(CurrentBackedgeReg, TRI) << ", BB#"
2312 << Exit->getNumber() << ")\n");
2313 } else {
2314 DEBUG(dbgs() << ")\n");
2315 }
2316 }
2317 }
2318
2319 void AMDGPUMachineCFGStructurizer::createEntryPHIs(LinearizedRegion *CurrentRegion) {
2320 DEBUG(PHIInfo.dump(MRI));
2321
2322 for (auto DRI = PHIInfo.dests_begin(), DE = PHIInfo.dests_end(); DRI != DE;
2323 ++DRI) {
2324
2325 unsigned DestReg = *DRI;
2326 createEntryPHI(CurrentRegion, DestReg);
2327 }
2328 PHIInfo.clear();
2329 }
2330
2331 void AMDGPUMachineCFGStructurizer::replaceRegisterWith(unsigned Register,
2332 unsigned NewRegister) {
2333 assert(Register != NewRegister && "Cannot replace a reg with itself");
2334
2335 for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(Register),
2336 E = MRI->reg_end();
2337 I != E;) {
2338 MachineOperand &O = *I;
2339 ++I;
2340 if (TargetRegisterInfo::isPhysicalRegister(NewRegister)) {
2341 DEBUG(dbgs() << "Trying to substitute physical register: "
2342 << PrintReg(NewRegister, MRI->getTargetRegisterInfo())
2343 << "\n");
2344 llvm_unreachable("Cannot substitute physical registers");
2345 // We don't handle physical registers, but if we need to
2346 // in the future This is how we do it:
2347 // O.substPhysReg(NewRegister, *TRI);
2348 } else {
2349 DEBUG(dbgs() << "Replacing register: "
2350 << PrintReg(Register, MRI->getTargetRegisterInfo())
2351 << " with "
2352 << PrintReg(NewRegister, MRI->getTargetRegisterInfo())
2353 << "\n");
2354 O.setReg(NewRegister);
2355 }
2356 }
2357 PHIInfo.deleteDef(Register);
2358
2359 getRegionMRT()->replaceLiveOutReg(Register, NewRegister);
2360
2361 DEBUG(PHIInfo.dump(MRI));
2362 }
2363
2364 void AMDGPUMachineCFGStructurizer::resolvePHIInfos(MachineBasicBlock *FunctionEntry) {
2365 DEBUG(dbgs() << "Resolve PHI Infos\n");
2366 DEBUG(PHIInfo.dump(MRI));
2367 for (auto DRI = PHIInfo.dests_begin(), DE = PHIInfo.dests_end(); DRI != DE;
2368 ++DRI) {
2369 unsigned DestReg = *DRI;
2370 DEBUG(dbgs() << "DestReg: " << PrintReg(DestReg, TRI) << "\n");
2371 auto SRI = PHIInfo.sources_begin(DestReg);
2372 unsigned SourceReg = (*SRI).first;
2373 DEBUG(dbgs() << "DestReg: " << PrintReg(DestReg, TRI)
2374 << " SourceReg: " << PrintReg(SourceReg, TRI) << "\n");
2375
2376 assert(PHIInfo.sources_end(DestReg) == ++SRI &&
2377 "More than one phi source in entry node");
2378 replaceRegisterWith(DestReg, SourceReg);
2379 }
2380 }
2381
2382 static bool isFunctionEntryBlock(MachineBasicBlock *MBB) {
2383 return ((&(*(MBB->getParent()->begin()))) == MBB);
2384 }
2385
2386 MachineBasicBlock *AMDGPUMachineCFGStructurizer::createIfRegion(
2387 MachineBasicBlock *MergeBB, MachineBasicBlock *CodeBB,
2388 LinearizedRegion *CurrentRegion, unsigned BBSelectRegIn,
2389 unsigned BBSelectRegOut) {
2390 if (isFunctionEntryBlock(CodeBB) && !CurrentRegion->getHasLoop()) {
2391 // Handle non-loop function entry block.
2392 // We need to allow loops to the entry block and then
2393 rewriteCodeBBTerminator(CodeBB, MergeBB, BBSelectRegOut);
2394 resolvePHIInfos(CodeBB);
2395 removeExternalCFGSuccessors(CodeBB);
2396 CodeBB->addSuccessor(MergeBB);
2397 CurrentRegion->addMBB(CodeBB);
2398 return nullptr;
2399 }
2400 if (CurrentRegion->getEntry() == CodeBB && !CurrentRegion->getHasLoop()) {
2401 // Handle non-loop region entry block.
2402 MachineFunction *MF = MergeBB->getParent();
2403 auto MergeIter = MergeBB->getIterator();
2404 auto CodeBBStartIter = CodeBB->getIterator();
2405 auto CodeBBEndIter = ++(CodeBB->getIterator());
2406 if (CodeBBEndIter != MergeIter) {
2407 MF->splice(MergeIter, CodeBBStartIter, CodeBBEndIter);
2408 }
2409 rewriteCodeBBTerminator(CodeBB, MergeBB, BBSelectRegOut);
2410 prunePHIInfo(CodeBB);
2411 createEntryPHIs(CurrentRegion);
2412 removeExternalCFGSuccessors(CodeBB);
2413 CodeBB->addSuccessor(MergeBB);
2414 CurrentRegion->addMBB(CodeBB);
2415 return nullptr;
2416 } else {
2417 // Handle internal block.
2418 const TargetRegisterClass *RegClass = MRI->getRegClass(BBSelectRegIn);
2419 unsigned CodeBBSelectReg = MRI->createVirtualRegister(RegClass);
2420 rewriteCodeBBTerminator(CodeBB, MergeBB, CodeBBSelectReg);
2421 bool IsRegionEntryBB = CurrentRegion->getEntry() == CodeBB;
2422 MachineBasicBlock *IfBB = createIfBlock(MergeBB, CodeBB, CodeBB, CodeBB,
2423 BBSelectRegIn, IsRegionEntryBB);
2424 CurrentRegion->addMBB(IfBB);
2425 // If this is the entry block we need to make the If block the new
2426 // linearized region entry.
2427 if (IsRegionEntryBB) {
2428 CurrentRegion->setEntry(IfBB);
2429
2430 if (CurrentRegion->getHasLoop()) {
2431 MachineBasicBlock *RegionExit = CurrentRegion->getExit();
2432 MachineBasicBlock *ETrueBB = nullptr;
2433 MachineBasicBlock *EFalseBB = nullptr;
2434 SmallVector ECond;
2435
2436 const DebugLoc &DL = DebugLoc();
2437 TII->analyzeBranch(*RegionExit, ETrueBB, EFalseBB, ECond);
2438 TII->removeBranch(*RegionExit);
2439
2440 // We need to create a backedge if there is a loop
2441 unsigned Reg = TII->insertNE(
2442 RegionExit, RegionExit->instr_end(), DL,
2443 CurrentRegion->getRegionMRT()->getInnerOutputRegister(),
2444 CurrentRegion->getRegionMRT()->getEntry()->getNumber());
2445 MachineOperand RegOp =
2446 MachineOperand::CreateReg(Reg, false, false, true);
2447 ArrayRef Cond(RegOp);
2448 DEBUG(dbgs() << "RegionExitReg: ");
2449 DEBUG(Cond[0].print(dbgs(), TRI));
2450 DEBUG(dbgs() << "\n");
2451 TII->insertBranch(*RegionExit, CurrentRegion->getEntry(), RegionExit,
2452 Cond, DebugLoc());
2453 RegionExit->addSuccessor(CurrentRegion->getEntry());
2454 }
2455 }
2456 CurrentRegion->addMBB(CodeBB);
2457 LinearizedRegion InnerRegion(CodeBB, MRI, TRI, PHIInfo);
2458
2459 InnerRegion.setParent(CurrentRegion);
2460 DEBUG(dbgs() << "Insert BB Select PHI (BB)\n");
2461 insertMergePHI(IfBB, CodeBB, MergeBB, BBSelectRegOut, BBSelectRegIn,
2462 CodeBBSelectReg);
2463 InnerRegion.addMBB(MergeBB);
2464
2465 DEBUG(InnerRegion.print(dbgs(), TRI));
2466 rewriteLiveOutRegs(IfBB, CodeBB, MergeBB, &InnerRegion, CurrentRegion);
2467 extractKilledPHIs(CodeBB);
2468 if (IsRegionEntryBB) {
2469 createEntryPHIs(CurrentRegion);
2470 }
2471 return IfBB;
2472 }
2473 }
2474
2475 MachineBasicBlock *AMDGPUMachineCFGStructurizer::createIfRegion(
2476 MachineBasicBlock *MergeBB, LinearizedRegion *InnerRegion,
2477 LinearizedRegion *CurrentRegion, MachineBasicBlock *SelectBB,
2478 unsigned BBSelectRegIn, unsigned BBSelectRegOut) {
2479 unsigned CodeBBSelectReg =
2480 InnerRegion->getRegionMRT()->getInnerOutputRegister();
2481 MachineBasicBlock *CodeEntryBB = InnerRegion->getEntry();
2482 MachineBasicBlock *CodeExitBB = InnerRegion->getExit();
2483 MachineBasicBlock *IfBB = createIfBlock(MergeBB, CodeEntryBB, CodeExitBB,
2484 SelectBB, BBSelectRegIn, true);
2485 CurrentRegion->addMBB(IfBB);
2486 bool isEntry = CurrentRegion->getEntry() == InnerRegion->getEntry();
2487 if (isEntry) {
2488
2489 if (CurrentRegion->getHasLoop()) {
2490 MachineBasicBlock *RegionExit = CurrentRegion->getExit();
2491 MachineBasicBlock *ETrueBB = nullptr;
2492 MachineBasicBlock *EFalseBB = nullptr;
2493 SmallVector ECond;
2494
2495 const DebugLoc &DL = DebugLoc();
2496 TII->analyzeBranch(*RegionExit, ETrueBB, EFalseBB, ECond);
2497 TII->removeBranch(*RegionExit);
2498
2499 // We need to create a backedge if there is a loop
2500 unsigned Reg =
2501 TII->insertNE(RegionExit, RegionExit->instr_end(), DL,
2502 CurrentRegion->getRegionMRT()->getInnerOutputRegister(),
2503 CurrentRegion->getRegionMRT()->getEntry()->getNumber());
2504 MachineOperand RegOp = MachineOperand::CreateReg(Reg, false, false, true);
2505 ArrayRef Cond(RegOp);
2506 DEBUG(dbgs() << "RegionExitReg: ");
2507 DEBUG(Cond[0].print(dbgs(), TRI));
2508 DEBUG(dbgs() << "\n");
2509 TII->insertBranch(*RegionExit, CurrentRegion->getEntry(), RegionExit,
2510 Cond, DebugLoc());
2511 RegionExit->addSuccessor(IfBB);
2512 }
2513 }
2514 CurrentRegion->addMBBs(InnerRegion);
2515 DEBUG(dbgs() << "Insert BB Select PHI (region)\n");
2516 insertMergePHI(IfBB, CodeExitBB, MergeBB, BBSelectRegOut, BBSelectRegIn,
2517 CodeBBSelectReg);
2518
2519 rewriteLiveOutRegs(IfBB, /* CodeEntryBB */ CodeExitBB, MergeBB, InnerRegion,
2520 CurrentRegion);
2521
2522 rewriteRegionEntryPHIs(InnerRegion, IfBB);
2523
2524 if (isEntry) {
2525 CurrentRegion->setEntry(IfBB);
2526 }
2527
2528 if (isEntry) {
2529 createEntryPHIs(CurrentRegion);
2530 }
2531
2532 return IfBB;
2533 }
2534
2535 void AMDGPUMachineCFGStructurizer::splitLoopPHI(MachineInstr &PHI,
2536 MachineBasicBlock *Entry,
2537 MachineBasicBlock *EntrySucc,
2538 LinearizedRegion *LRegion) {
2539 SmallVector PHIRegionIndices;
2540 getPHIRegionIndices(LRegion, PHI, PHIRegionIndices);
2541
2542 assert(PHIRegionIndices.size() == 1);
2543
2544 unsigned RegionIndex = PHIRegionIndices[0];
2545 unsigned RegionSourceReg = getPHISourceReg(PHI, RegionIndex);
2546 MachineBasicBlock *RegionSourceMBB = getPHIPred(PHI, RegionIndex);
2547 unsigned PHIDest = getPHIDestReg(PHI);
2548 unsigned PHISource = PHIDest;
2549 unsigned ReplaceReg;
2550
2551 if (shrinkPHI(PHI, PHIRegionIndices, &ReplaceReg)) {
2552 PHISource = ReplaceReg;
2553 }
2554
2555 const TargetRegisterClass *RegClass = MRI->getRegClass(PHIDest);
2556 unsigned NewDestReg = MRI->createVirtualRegister(RegClass);
2557 LRegion->replaceRegisterInsideRegion(PHIDest, NewDestReg, false, MRI);
2558 MachineInstrBuilder MIB =
2559 BuildMI(*EntrySucc, EntrySucc->instr_begin(), PHI.getDebugLoc(),
2560 TII->get(TargetOpcode::PHI), NewDestReg);
2561 DEBUG(dbgs() << "Split Entry PHI " << PrintReg(NewDestReg, TRI)
2562 << " = PHI(");
2563 MIB.addReg(PHISource);
2564 MIB.addMBB(Entry);
2565 DEBUG(dbgs() << PrintReg(PHISource, TRI) << ", BB#" << Entry->getNumber());
2566 MIB.addReg(RegionSourceReg);
2567 MIB.addMBB(RegionSourceMBB);
2568 DEBUG(dbgs() << " ," << PrintReg(RegionSourceReg, TRI) << ", BB#"
2569 << RegionSourceMBB->getNumber() << ")\n");
2570 }
2571
2572 void AMDGPUMachineCFGStructurizer::splitLoopPHIs(MachineBasicBlock *Entry,
2573 MachineBasicBlock *EntrySucc,
2574 LinearizedRegion *LRegion) {
2575 SmallVector PHIs;
2576 collectPHIs(Entry, PHIs);
2577
2578 for (auto PHII : PHIs) {
2579 splitLoopPHI(*PHII, Entry, EntrySucc, LRegion);
2580 }
2581 }
2582
2583 // Split the exit block so that we can insert a end control flow
2584 MachineBasicBlock *
2585 AMDGPUMachineCFGStructurizer::splitExit(LinearizedRegion *LRegion) {
2586 auto MRTRegion = LRegion->getRegionMRT();
2587 auto Exit = LRegion->getExit();
2588 auto MF = Exit->getParent();
2589 auto Succ = MRTRegion->getSucc();
2590
2591 auto NewExit = MF->CreateMachineBasicBlock();
2592 auto AfterExitIter = Exit->getIterator();
2593 AfterExitIter++;
2594 MF->insert(AfterExitIter, NewExit);
2595 Exit->removeSuccessor(Succ);
2596 Exit->addSuccessor(NewExit);
2597 NewExit->addSuccessor(Succ);
2598 insertUnconditionalBranch(NewExit, Succ);
2599 LRegion->addMBB(NewExit);
2600 LRegion->setExit(NewExit);
2601
2602 DEBUG(dbgs() << "Created new exit block: " << NewExit->getNumber() << "\n");
2603
2604 // Replace any PHI Predecessors in the successor with NewExit
2605 for (auto &II : *Succ) {
2606 MachineInstr &Instr = II;
2607
2608 // If we are past the PHI instructions we are done
2609 if (!Instr.isPHI())
2610 break;
2611
2612 int numPreds = getPHINumInputs(Instr);
2613 for (int i = 0; i < numPreds; ++i) {
2614 auto Pred = getPHIPred(Instr, i);
2615 if (Pred == Exit) {
2616 setPhiPred(Instr, i, NewExit);
2617 }
2618 }
2619 }
2620
2621 return NewExit;
2622 }
2623
2624
2625 static MachineBasicBlock *split(MachineBasicBlock::iterator I) {
2626 // Create the fall-through block.
2627 MachineBasicBlock *MBB = (*I).getParent();
2628 MachineFunction *MF = MBB->getParent();
2629 MachineBasicBlock *SuccMBB = MF->CreateMachineBasicBlock();
2630 auto MBBIter = ++(MBB->getIterator());
2631 MF->insert(MBBIter, SuccMBB);
2632 SuccMBB->transferSuccessorsAndUpdatePHIs(MBB);
2633 MBB->addSuccessor(SuccMBB);
2634
2635 // Splice the code over.
2636 SuccMBB->splice(SuccMBB->end(), MBB, I, MBB->end());
2637
2638 return SuccMBB;
2639 }
2640
2641 // Split the entry block separating PHI-nodes and the rest of the code
2642 // This is needed to insert an initializer for the bb select register
2643 // inloop regions.
2644
2645 MachineBasicBlock *
2646 AMDGPUMachineCFGStructurizer::splitEntry(LinearizedRegion *LRegion) {
2647 MachineBasicBlock *Entry = LRegion->getEntry();
2648 MachineBasicBlock *EntrySucc = split(Entry->getFirstNonPHI());
2649 MachineBasicBlock *Exit = LRegion->getExit();
2650
2651 DEBUG(dbgs() << "Split BB#" << Entry->getNumber() << " to BB#"
2652 << Entry->getNumber() << " -> BB#" << EntrySucc->getNumber()
2653 << "\n");
2654 LRegion->addMBB(EntrySucc);
2655
2656 // Make the backedge go to Entry Succ
2657 if (Exit->isSuccessor(Entry)) {
2658 Exit->removeSuccessor(Entry);
2659 }
2660 Exit->addSuccessor(EntrySucc);
2661 MachineInstr &Branch = *(Exit->instr_rbegin());
2662 for (auto &UI : Branch.uses()) {
2663 if (UI.isMBB() && UI.getMBB() == Entry) {
2664 UI.setMBB(EntrySucc);
2665 }
2666 }
2667
2668 splitLoopPHIs(Entry, EntrySucc, LRegion);
2669
2670 return EntrySucc;
2671 }
2672
2673 LinearizedRegion *
2674 AMDGPUMachineCFGStructurizer::initLinearizedRegion(RegionMRT *Region) {
2675 LinearizedRegion *LRegion = Region->getLinearizedRegion();
2676 LRegion->initLiveOut(Region, MRI, TRI, PHIInfo);
2677 LRegion->setEntry(Region->getEntry());
2678 return LRegion;
2679 }
2680
2681 static void removeOldExitPreds(RegionMRT *Region) {
2682 MachineBasicBlock *Exit = Region->getSucc();
2683 if (Exit == nullptr) {
2684 return;
2685 }
2686 for (MachineBasicBlock::pred_iterator PI = Exit->pred_begin(),
2687 E = Exit->pred_end();
2688 PI != E; ++PI) {
2689 if (Region->contains(*PI)) {
2690 (*PI)->removeSuccessor(Exit);
2691 }
2692 }
2693 }
2694
2695 static bool mbbHasBackEdge(MachineBasicBlock *MBB,
2696 SmallPtrSet &MBBs) {
2697 for (auto SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) {
2698 if (MBBs.count(*SI) != 0) {
2699 return true;
2700 }
2701 }
2702 return false;
2703 }
2704
2705 static bool containsNewBackedge(MRT *Tree,
2706 SmallPtrSet &MBBs) {
2707 // Need to traverse this in reverse since it is in post order.
2708 if (Tree == nullptr)
2709 return false;
2710
2711 if (Tree->isMBB()) {
2712 MachineBasicBlock *MBB = Tree->getMBBMRT()->getMBB();
2713 MBBs.insert(MBB);
2714 if (mbbHasBackEdge(MBB, MBBs)) {
2715 return true;
2716 }
2717 } else {
2718 RegionMRT *Region = Tree->getRegionMRT();
2719 SetVector *Children = Region->getChildren();
2720 for (auto CI = Children->rbegin(), CE = Children->rend(); CI != CE; ++CI) {
2721 if (containsNewBackedge(*CI, MBBs))
2722 return true;
2723 }
2724 }
2725 return false;
2726 }
2727
2728 static bool containsNewBackedge(RegionMRT *Region) {
2729 SmallPtrSet MBBs;
2730 return containsNewBackedge(Region, MBBs);
2731 }
2732
2733 bool AMDGPUMachineCFGStructurizer::structurizeComplexRegion(RegionMRT *Region) {
2734 auto *LRegion = initLinearizedRegion(Region);
2735 LRegion->setHasLoop(containsNewBackedge(Region));
2736 MachineBasicBlock *LastMerge = createLinearizedExitBlock(Region);
2737 MachineBasicBlock *CurrentMerge = LastMerge;
2738 LRegion->addMBB(LastMerge);
2739 LRegion->setExit(LastMerge);
2740
2741 rewriteRegionExitPHIs(Region, LastMerge, LRegion);
2742 removeOldExitPreds(Region);
2743
2744 DEBUG(PHIInfo.dump(MRI));
2745
2746 auto Entry = Region->getEntry();
2747
2748 SetVector *Children = Region->getChildren();
2749 DEBUG(dbgs() << "===========If Region Start===============\n");
2750 if (LRegion->getHasLoop()) {
2751 DEBUG(dbgs() << "Has Backedge: Yes\n");
2752 } else {
2753 DEBUG(dbgs() << "Has Backedge: No\n");
2754 }
2755
2756 unsigned BBSelectRegIn;
2757 unsigned BBSelectRegOut;
2758 for (auto CI = Children->begin(), CE = Children->end(); CI != CE; ++CI) {
2759 DEBUG(dbgs() << "CurrentRegion: \n");
2760 DEBUG(LRegion->print(dbgs(), TRI));
2761
2762 auto CNI = CI;
2763 ++CNI;
2764
2765 MRT *Child = (*CI);
2766
2767 if (Child->isRegion()) {
2768
2769 LinearizedRegion *InnerLRegion =
2770 Child->getRegionMRT()->getLinearizedRegion();
2771 // We found the block is the exit of an inner region, we need
2772 // to put it in the current linearized region.
2773
2774 DEBUG(dbgs() << "Linearizing region: ");
2775 DEBUG(InnerLRegion->print(dbgs(), TRI));
2776 DEBUG(dbgs() << "\n");
2777
2778 MachineBasicBlock *InnerEntry = InnerLRegion->getEntry();
2779 if ((&(*(InnerEntry->getParent()->begin()))) == InnerEntry) {
2780 // Entry has already been linearized, no need to do this region.
2781 unsigned OuterSelect = InnerLRegion->getBBSelectRegOut();
2782 unsigned InnerSelectReg =
2783 InnerLRegion->getRegionMRT()->getInnerOutputRegister();
2784 replaceRegisterWith(InnerSelectReg, OuterSelect),
2785 resolvePHIInfos(InnerEntry);
2786 if (!InnerLRegion->getExit()->isSuccessor(CurrentMerge))
2787 InnerLRegion->getExit()->addSuccessor(CurrentMerge);
2788 continue;
2789 }
2790
2791 BBSelectRegOut = Child->getBBSelectRegOut();
2792 BBSelectRegIn = Child->getBBSelectRegIn();
2793
2794 DEBUG(dbgs() << "BBSelectRegIn: " << PrintReg(BBSelectRegIn, TRI)
2795 << "\n");
2796 DEBUG(dbgs() << "BBSelectRegOut: " << PrintReg(BBSelectRegOut, TRI)
2797 << "\n");
2798
2799 MachineBasicBlock *IfEnd = CurrentMerge;
2800 CurrentMerge = createIfRegion(CurrentMerge, InnerLRegion, LRegion,
2801 Child->getRegionMRT()->getEntry(),
2802 BBSelectRegIn, BBSelectRegOut);
2803 TII->convertNonUniformIfRegion(CurrentMerge, IfEnd);
2804 } else {
2805 MachineBasicBlock *MBB = Child->getMBBMRT()->getMBB();
2806 DEBUG(dbgs() << "Linearizing block: " << MBB->getNumber() << "\n");
2807
2808 if (MBB == getSingleExitNode(*(MBB->getParent()))) {
2809 // If this is the exit block then we need to skip to the next.
2810 // The "in" register will be transferred to "out" in the next
2811 // iteration.
2812 continue;
2813 }
2814
2815 BBSelectRegOut = Child->getBBSelectRegOut();
2816 BBSelectRegIn = Child->getBBSelectRegIn();
2817
2818 DEBUG(dbgs() << "BBSelectRegIn: " << PrintReg(BBSelectRegIn, TRI)
2819 << "\n");
2820 DEBUG(dbgs() << "BBSelectRegOut: " << PrintReg(BBSelectRegOut, TRI)
2821 << "\n");
2822
2823 MachineBasicBlock *IfEnd = CurrentMerge;
2824 // This is a basic block that is not part of an inner region, we
2825 // need to put it in the current linearized region.
2826 CurrentMerge = createIfRegion(CurrentMerge, MBB, LRegion, BBSelectRegIn,
2827 BBSelectRegOut);
2828 if (CurrentMerge) {
2829 TII->convertNonUniformIfRegion(CurrentMerge, IfEnd);
2830 }
2831
2832 DEBUG(PHIInfo.dump(MRI));
2833 }
2834 }
2835
2836 LRegion->removeFalseRegisterKills(MRI);
2837
2838 if (LRegion->getHasLoop()) {
2839 MachineBasicBlock *NewSucc = splitEntry(LRegion);
2840 if (isFunctionEntryBlock(LRegion->getEntry())) {
2841 resolvePHIInfos(LRegion->getEntry());
2842 }
2843 const DebugLoc &DL = NewSucc->findDebugLoc(NewSucc->getFirstNonPHI());
2844 unsigned InReg = LRegion->getBBSelectRegIn();
2845 unsigned InnerSelectReg =
2846 MRI->createVirtualRegister(MRI->getRegClass(InReg));
2847 unsigned NewInReg = MRI->createVirtualRegister(MRI->getRegClass(InReg));
2848 TII->materializeImmediate(*(LRegion->getEntry()),
2849 LRegion->getEntry()->getFirstTerminator(), DL,
2850 NewInReg, Region->getEntry()->getNumber());
2851 // Need to be careful about updating the registers inside the region.
2852 LRegion->replaceRegisterInsideRegion(InReg, InnerSelectReg, false, MRI);
2853 DEBUG(dbgs() << "Loop BBSelect Merge PHI:\n");
2854 insertMergePHI(LRegion->getEntry(), LRegion->getExit(), NewSucc,
2855 InnerSelectReg, NewInReg,
2856 LRegion->getRegionMRT()->getInnerOutputRegister());
2857 splitExit(LRegion);
2858 TII->convertNonUniformLoopRegion(NewSucc, LastMerge);
2859 }
2860
2861 if (Region->isRoot()) {
2862 TII->insertReturn(*LastMerge);
2863 }
2864
2865 DEBUG(Entry->getParent()->dump());
2866 DEBUG(LRegion->print(dbgs(), TRI));
2867 DEBUG(PHIInfo.dump(MRI));
2868
2869 DEBUG(dbgs() << "===========If Region End===============\n");
2870
2871 Region->setLinearizedRegion(LRegion);
2872 return true;
2873 }
2874
2875 bool AMDGPUMachineCFGStructurizer::structurizeRegion(RegionMRT *Region) {
2876 if (false && regionIsSimpleIf(Region)) {
2877 transformSimpleIfRegion(Region);
2878 return true;
2879 } else if (regionIsSequence(Region)) {
2880 fixupRegionExits(Region);
2881 return false;
2882 } else {
2883 structurizeComplexRegion(Region);
2884 }
2885 return false;
2886 }
2887
2888 static int structurize_once = 0;
2889
2890 bool AMDGPUMachineCFGStructurizer::structurizeRegions(RegionMRT *Region,
2891 bool isTopRegion) {
2892 bool Changed = false;
2893
2894 auto Children = Region->getChildren();
2895 for (auto CI : *Children) {
2896 if (CI->isRegion()) {
2897 Changed |= structurizeRegions(CI->getRegionMRT(), false);
2898 }
2899 }
2900
2901 if (structurize_once < 2 || true) {
2902 Changed |= structurizeRegion(Region);
2903 structurize_once++;
2904 }
2905 return Changed;
2906 }
2907
2908 void AMDGPUMachineCFGStructurizer::initFallthroughMap(MachineFunction &MF) {
2909 DEBUG(dbgs() << "Fallthrough Map:\n");
2910 for (auto &MBBI : MF) {
2911 MachineBasicBlock *MBB = MBBI.getFallThrough();
2912 if (MBB != nullptr) {
2913 DEBUG(dbgs() << "Fallthrough: " << MBBI.getNumber() << " -> "
2914 << MBB->getNumber() << "\n");
2915 }
2916 FallthroughMap[&MBBI] = MBB;
2917 }
2918 }
2919
2920 void AMDGPUMachineCFGStructurizer::createLinearizedRegion(RegionMRT *Region,
2921 unsigned SelectOut) {
2922 LinearizedRegion *LRegion = new LinearizedRegion();
2923 if (SelectOut) {
2924 LRegion->addLiveOut(SelectOut);
2925 DEBUG(dbgs() << "Add LiveOut (BBSelect): " << PrintReg(SelectOut, TRI)
2926 << "\n");
2927 }
2928 LRegion->setRegionMRT(Region);
2929 Region->setLinearizedRegion(LRegion);
2930 LRegion->setParent(Region->getParent()
2931 ? Region->getParent()->getLinearizedRegion()
2932 : nullptr);
2933 }
2934
2935 unsigned
2936 AMDGPUMachineCFGStructurizer::initializeSelectRegisters(MRT *MRT, unsigned SelectOut,
2937 MachineRegisterInfo *MRI,
2938 const SIInstrInfo *TII) {
2939 if (MRT->isRegion()) {
2940 RegionMRT *Region = MRT->getRegionMRT();
2941 Region->setBBSelectRegOut(SelectOut);
2942 unsigned InnerSelectOut = createBBSelectReg(TII, MRI);
2943
2944 // Fixme: Move linearization creation to the original spot
2945 createLinearizedRegion(Region, SelectOut);
2946
2947 for (auto CI = Region->getChildren()->begin(),
2948 CE = Region->getChildren()->end();
2949 CI != CE; ++CI) {
2950 InnerSelectOut =
2951 initializeSelectRegisters((*CI), InnerSelectOut, MRI, TII);
2952 }
2953 MRT->setBBSelectRegIn(InnerSelectOut);
2954 return InnerSelectOut;
2955 } else {
2956 MRT->setBBSelectRegOut(SelectOut);
2957 unsigned NewSelectIn = createBBSelectReg(TII, MRI);
2958 MRT->setBBSelectRegIn(NewSelectIn);
2959 return NewSelectIn;
2960 }
2961 }
2962
2963 static void checkRegOnlyPHIInputs(MachineFunction &MF) {
2964 for (auto &MBBI : MF) {
2965 for (MachineBasicBlock::instr_iterator I = MBBI.instr_begin(),
2966 E = MBBI.instr_end();
2967 I != E; ++I) {
2968 MachineInstr &Instr = *I;
2969 if (Instr.isPHI()) {
2970 int numPreds = getPHINumInputs(Instr);
2971 for (int i = 0; i < numPreds; ++i) {
2972 assert(Instr.getOperand(i * 2 + 1).isReg() &&
2973 "PHI Operand not a register");
2974 }
2975 }
2976 }
2977 }
2978 }
2979
2980
2981 INITIALIZE_PASS_BEGIN(AMDGPUMachineCFGStructurizer, "amdgpu-machine-cfg-structurizer",
2982 "AMDGPU Machine CFG Structurizer", false, false)
2983 INITIALIZE_PASS_DEPENDENCY(MachineRegionInfoPass)
2984 INITIALIZE_PASS_END(AMDGPUMachineCFGStructurizer, "amdgpu-machine-cfg-structurizer",
2985 "AMDGPU Machine CFG Structurizer", false, false)
2986
2987 char AMDGPUMachineCFGStructurizerID = AMDGPUMachineCFGStructurizer::ID;
2988
2989
2990 bool AMDGPUMachineCFGStructurizer::runOnMachineFunction(MachineFunction &MF) {
2991 const SISubtarget &ST = MF.getSubtarget();
2992 const SIInstrInfo *TII = ST.getInstrInfo();
2993 TRI = ST.getRegisterInfo();
2994 MRI = &(MF.getRegInfo());
2995 initFallthroughMap(MF);
2996
2997 checkRegOnlyPHIInputs(MF);
2998 DEBUG(dbgs() << "----STRUCTURIZER START----\n");
2999 DEBUG(MF.dump());
3000
3001 Regions = &(getAnalysis().getRegionInfo());
3002 DEBUG(Regions->dump());
3003
3004 RegionMRT *RTree = MRT::buildMRT(MF, Regions, TII, MRI);
3005 setRegionMRT(RTree);
3006 initializeSelectRegisters(RTree, 0, MRI, TII);
3007 DEBUG(RTree->dump(TRI));
3008 bool result = structurizeRegions(RTree, true);
3009 delete RTree;
3010 DEBUG(dbgs() << "----STRUCTURIZER END----\n");
3011 initFallthroughMap(MF);
3012 return result;
3013 }
3014
3015 FunctionPass *llvm::createAMDGPUMachineCFGStructurizerPass() {
3016 return new AMDGPUMachineCFGStructurizer();
3017 }
3018
117117 cl::desc("Use new waitcnt insertion pass"),
118118 cl::init(false));
119119
120 // Option to run late CFG structurizer
121 static cl::opt LateCFGStructurize(
122 "amdgpu-late-structurize",
123 cl::desc("Enable late CFG structurization"),
124 cl::init(false),
125 cl::Hidden);
126
127120 extern "C" void LLVMInitializeAMDGPUTarget() {
128121 // Register the target
129122 RegisterTargetMachine X(getTheAMDGPUTarget());
708701 // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
709702 // regions formed by them.
710703 addPass(&AMDGPUUnifyDivergentExitNodesID);
711 if (!LateCFGStructurize) {
712 addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
713 }
704 addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
714705 addPass(createSinkingPass());
715706 addPass(createSITypeRewriter());
716707 addPass(createAMDGPUAnnotateUniformValues());
717 if (!LateCFGStructurize) {
718 addPass(createSIAnnotateControlFlowPass());
719 }
708 addPass(createSIAnnotateControlFlowPass());
720709
721710 return false;
722711 }
780769 #endif
781770
782771 void GCNPassConfig::addPreRegAlloc() {
783 if (LateCFGStructurize) {
784 addPass(createAMDGPUMachineCFGStructurizerPass());
785 }
786772 addPass(createSIWholeQuadModePass());
787773 }
788774
4747 AMDGPUISelDAGToDAG.cpp
4848 AMDGPULowerIntrinsics.cpp
4949 AMDGPUMCInstLower.cpp
50 AMDGPUMachineCFGStructurizer.cpp
5150 AMDGPUMachineFunction.cpp
5251 AMDGPUUnifyMetadata.cpp
5352 AMDGPUOpenCLImageTypeLoweringPass.cpp
493493 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
494494
495495 return Opcode;
496 }
497
498 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
499 MachineBasicBlock::iterator MI,
500 const DebugLoc &DL, unsigned DestReg,
501 int64_t Value) const {
502 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
503 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
504 if (RegClass == &AMDGPU::SReg_32RegClass ||
505 RegClass == &AMDGPU::SGPR_32RegClass ||
506 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
507 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
508 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
509 .addImm(Value);
510 return;
511 }
512
513 if (RegClass == &AMDGPU::SReg_64RegClass ||
514 RegClass == &AMDGPU::SGPR_64RegClass ||
515 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
516 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
517 .addImm(Value);
518 return;
519 }
520
521 if (RegClass == &AMDGPU::VGPR_32RegClass) {
522 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
523 .addImm(Value);
524 return;
525 }
526 if (RegClass == &AMDGPU::VReg_64RegClass) {
527 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
528 .addImm(Value);
529 return;
530 }
531
532 unsigned EltSize = 4;
533 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
534 if (RI.isSGPRClass(RegClass)) {
535 if (RI.getRegSizeInBits(*RegClass) > 32) {
536 Opcode = AMDGPU::S_MOV_B64;
537 EltSize = 8;
538 } else {
539 Opcode = AMDGPU::S_MOV_B32;
540 EltSize = 4;
541 }
542 }
543
544 ArrayRef SubIndices = RI.getRegSplitParts(RegClass, EltSize);
545 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
546 int64_t IdxValue = Idx == 0 ? Value : 0;
547
548 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
549 get(Opcode), RI.getSubReg(DestReg, Idx));
550 Builder.addImm(IdxValue);
551 }
552 }
553
554 const TargetRegisterClass *
555 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
556 return &AMDGPU::VGPR_32RegClass;
557 }
558
559 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
560 MachineBasicBlock::iterator I,
561 const DebugLoc &DL, unsigned DstReg,
562 ArrayRef Cond,
563 unsigned TrueReg,
564 unsigned FalseReg) const {
565 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
566 const TargetRegisterClass *RegClass = MRI.getRegClass(DstReg);
567 assert(RegClass == &AMDGPU::VGPR_32RegClass && "Not a VGPR32 reg");
568
569 if (Cond.size() == 1) {
570 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
571 .addReg(FalseReg)
572 .addReg(TrueReg)
573 .add(Cond[0]);
574 } else if (Cond.size() == 2) {
575 assert(Cond[0].isImm() && "Cond[0] is not an immediate");
576 switch (Cond[0].getImm()) {
577 case SIInstrInfo::SCC_TRUE: {
578 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
579 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
580 .addImm(-1)
581 .addImm(0);
582 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
583 .addReg(FalseReg)
584 .addReg(TrueReg)
585 .addReg(SReg);
586 break;
587 }
588 case SIInstrInfo::SCC_FALSE: {
589 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
590 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
591 .addImm(0)
592 .addImm(-1);
593 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
594 .addReg(FalseReg)
595 .addReg(TrueReg)
596 .addReg(SReg);
597 break;
598 }
599 case SIInstrInfo::VCCNZ: {
600 MachineOperand RegOp = Cond[1];
601 RegOp.setImplicit(false);
602 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
603 .addReg(FalseReg)
604 .addReg(TrueReg)
605 .add(RegOp);
606 break;
607 }
608 case SIInstrInfo::VCCZ: {
609 MachineOperand RegOp = Cond[1];
610 RegOp.setImplicit(false);
611 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
612 .addReg(TrueReg)
613 .addReg(FalseReg)
614 .add(RegOp);
615 break;
616 }
617 case SIInstrInfo::EXECNZ: {
618 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
619 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
620 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
621 .addImm(0);
622 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
623 .addImm(-1)
624 .addImm(0);
625 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
626 .addReg(FalseReg)
627 .addReg(TrueReg)
628 .addReg(SReg);
629 break;
630 }
631 case SIInstrInfo::EXECZ: {
632 unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
633 unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
634 BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
635 .addImm(0);
636 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
637 .addImm(0)
638 .addImm(-1);
639 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
640 .addReg(FalseReg)
641 .addReg(TrueReg)
642 .addReg(SReg);
643 llvm_unreachable("Unhandled branch predicate EXECZ");
644 break;
645 }
646 default:
647 llvm_unreachable("invalid branch predicate");
648 }
649 } else {
650 llvm_unreachable("Can only handle Cond size 1 or 2");
651 }
652 }
653
654 unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
655 MachineBasicBlock::iterator I,
656 const DebugLoc &DL,
657 unsigned SrcReg, int Value) const {
658 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
659 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
660 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
661 .addImm(Value)
662 .addReg(SrcReg);
663
664 return Reg;
665 }
666
667 unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
668 MachineBasicBlock::iterator I,
669 const DebugLoc &DL,
670 unsigned SrcReg, int Value) const {
671 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
672 unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
673 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
674 .addImm(Value)
675 .addReg(SrcReg);
676
677 return Reg;
678496 }
679497
680498 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1013831 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1014832 MachineBasicBlock::iterator MI) const {
1015833 insertWaitStates(MBB, MI, 1);
1016 }
1017
1018 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1019 auto MF = MBB.getParent();
1020 SIMachineFunctionInfo *Info = MF->getInfo();
1021
1022 assert(Info->isEntryFunction());
1023
1024 if (MBB.succ_empty()) {
1025 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1026 if (HasNoTerminator)
1027 BuildMI(MBB, MBB.end(), DebugLoc(),
1028 get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG));
1029 }
1030834 }
1031835
1032836 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) const {
14361240 return false;
14371241 }
14381242
1439 MachineBasicBlock *CondBB = nullptr;
1440
1441 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1442 CondBB = I->getOperand(1).getMBB();
1443 Cond.push_back(I->getOperand(0));
1444 } else {
1445 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1446 if (Pred == INVALID_BR)
1447 return true;
1448
1449 CondBB = I->getOperand(0).getMBB();
1450 Cond.push_back(MachineOperand::CreateImm(Pred));
1451 Cond.push_back(I->getOperand(1)); // Save the branch register.
1452 }
1243 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1244 if (Pred == INVALID_BR)
1245 return true;
1246
1247 MachineBasicBlock *CondBB = I->getOperand(0).getMBB();
1248 Cond.push_back(MachineOperand::CreateImm(Pred));
1249 Cond.push_back(I->getOperand(1)); // Save the branch register.
1250
14531251 ++I;
14541252
14551253 if (I == MBB.end()) {
15521350 return 1;
15531351 }
15541352
1555 if(Cond.size() == 1 && Cond[0].isReg()) {
1556 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1557 .add(Cond[0])
1558 .addMBB(TBB);
1559 return 1;
1560 }
1561
15621353 assert(TBB && Cond[0].isImm());
15631354
15641355 unsigned Opcode
15981389
15991390 bool SIInstrInfo::reverseBranchCondition(
16001391 SmallVectorImpl &Cond) const {
1601 if (Cond.size() != 2) {
1602 return true;
1603 }
1604
1605 if (Cond[0].isImm()) {
1606 Cond[0].setImm(-Cond[0].getImm());
1607 return false;
1608 }
1609
1610 return true;
1392 assert(Cond.size() == 2);
1393 Cond[0].setImm(-Cond[0].getImm());
1394 return false;
16111395 }
16121396
16131397 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
41353919 return false;
41363920 }
41373921
4138 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
4139 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
4140 }
4141
4142 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
4143 MachineBasicBlock *IfEnd) const {
4144 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
4145 assert(TI != IfEntry->end());
4146
4147 MachineInstr *Branch = &(*TI);
4148 MachineFunction *MF = IfEntry->getParent();
4149 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
4150
4151 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
4152 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4153 MachineInstr *SIIF =
4154 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
4155 .add(Branch->getOperand(0))
4156 .add(Branch->getOperand(1));
4157 MachineInstr *SIEND =
4158 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
4159 .addReg(DstReg);
4160
4161 IfEntry->erase(TI);
4162 IfEntry->insert(IfEntry->end(), SIIF);
4163 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
4164 }
4165 }
4166
4167 void SIInstrInfo::convertNonUniformLoopRegion(
4168 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
4169 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
4170 // We expect 2 terminators, one conditional and one unconditional.
4171 assert(TI != LoopEnd->end());
4172
4173 MachineInstr *Branch = &(*TI);
4174 MachineFunction *MF = LoopEnd->getParent();
4175 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
4176
4177 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
4178
4179 unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4180 unsigned BackEdgeReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4181 MachineInstrBuilder HeaderPHIBuilder =
4182 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
4183 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(),
4184 E = LoopEntry->pred_end();
4185 PI != E; ++PI) {
4186 if (*PI == LoopEnd) {
4187 HeaderPHIBuilder.addReg(BackEdgeReg);
4188 } else {
4189 MachineBasicBlock *PMBB = *PI;
4190 unsigned ZeroReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4191 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(),
4192 ZeroReg, 0);
4193 HeaderPHIBuilder.addReg(ZeroReg);
4194 }
4195 HeaderPHIBuilder.addMBB(*PI);
4196 }
4197 MachineInstr *HeaderPhi = HeaderPHIBuilder;
4198 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
4199 get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
4200 .addReg(DstReg)
4201 .add(Branch->getOperand(0));
4202 MachineInstr *SILOOP =
4203 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
4204 .addReg(BackEdgeReg)
4205 .addMBB(LoopEntry);
4206
4207 LoopEntry->insert(LoopEntry->begin(), HeaderPhi);
4208 LoopEnd->erase(TI);
4209 LoopEnd->insert(LoopEnd->end(), SIIFBREAK);
4210 LoopEnd->insert(LoopEnd->end(), SILOOP);
4211 }
4212 }
4213
42143922 ArrayRef>
42153923 SIInstrInfo::getSerializableTargetIndices() const {
42163924 static const std::pair TargetIndices[] = {
142142 RegScavenger *RS, unsigned TmpReg,
143143 unsigned Offset, unsigned Size) const;
144144
145 void materializeImmediate(MachineBasicBlock &MBB,
146 MachineBasicBlock::iterator MI,
147 const DebugLoc &DL,
148 unsigned DestReg,
149 int64_t Value) const;
150
151 const TargetRegisterClass *getPreferredSelectRegClass(
152 unsigned Size) const;
153
154 unsigned insertNE(MachineBasicBlock *MBB,
155 MachineBasicBlock::iterator I, const DebugLoc &DL,
156 unsigned SrcReg, int Value) const;
157
158 unsigned insertEQ(MachineBasicBlock *MBB,
159 MachineBasicBlock::iterator I, const DebugLoc &DL,
160 unsigned SrcReg, int Value) const;
161
162145 void storeRegToStackSlot(MachineBasicBlock &MBB,
163146 MachineBasicBlock::iterator MI, unsigned SrcReg,
164147 bool isKill, int FrameIndex,
209192 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
210193 MachineBasicBlock *&FBB,
211194 SmallVectorImpl &Cond,
212 bool AllowModify = false) const override;
195 bool AllowModify) const override;
213196
214197 unsigned removeBranch(MachineBasicBlock &MBB,
215198 int *BytesRemoved = nullptr) const override;
233216 MachineBasicBlock::iterator I, const DebugLoc &DL,
234217 unsigned DstReg, ArrayRef Cond,
235218 unsigned TrueReg, unsigned FalseReg) const override;
236
237 void insertVectorSelect(MachineBasicBlock &MBB,
238 MachineBasicBlock::iterator I, const DebugLoc &DL,
239 unsigned DstReg, ArrayRef Cond,
240 unsigned TrueReg, unsigned FalseReg) const;
241219
242220 bool
243221 areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb,
726704 void insertNoop(MachineBasicBlock &MBB,
727705 MachineBasicBlock::iterator MI) const override;
728706
729 void insertReturn(MachineBasicBlock &MBB) const;
730707 /// \brief Return the number of wait states that result from executing this
731708 /// instruction.
732709 unsigned getNumWaitStates(const MachineInstr &MI) const;
771748 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
772749
773750 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
774
775 bool isNonUniformBranchInstr(MachineInstr &Instr) const;
776
777 void convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
778 MachineBasicBlock *IfEnd) const;
779
780 void convertNonUniformLoopRegion(MachineBasicBlock *LoopEntry,
781 MachineBasicBlock *LoopEnd) const;
782751
783752 ArrayRef>
784753 getSerializableTargetIndices() const override;
172172 }
173173
174174 let isTerminator = 1 in {
175
176 def SI_NON_UNIFORM_BRCOND_PSEUDO : CFPseudoInstSI <
177 (outs),
178 (ins SReg_64:$vcc, brtarget:$target),
179 [(brcond i1:$vcc, bb:$target)]> {
180 let Size = 12;
181 }
182175
183176 def SI_IF: CFPseudoInstSI <
184177 (outs SReg_64:$dst), (ins SReg_64:$vcc, brtarget:$target),