llvm.org GIT mirror llvm / e2f9d30
Improve scheduling with branch coalescing This patch adds a MachineSSA pass that coalesces blocks that branch on the same condition. Committing on behalf of Lei Huang. Differential Revision: https://reviews.llvm.org/D28249 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@296670 91177308-0d34-0410-b5e6-96231b3b80d8 Nemanja Ivanovic 3 years ago
7 changed file(s) with 799 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
401401
402402 /// This pass frees the memory occupied by the MachineFunction.
403403 FunctionPass *createFreeMachineFunctionPass();
404
405 /// This pass combine basic blocks guarded by the same branch.
406 extern char &BranchCoalescingID;
404407 } // End llvm namespace
405408
406409 /// Target machine pass initializer for passes with dependencies. Use with
7676 void initializeBlockExtractorPassPass(PassRegistry&);
7777 void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&);
7878 void initializeBoundsCheckingPass(PassRegistry&);
79 void initializeBranchCoalescingPass(PassRegistry&);
7980 void initializeBranchFolderPassPass(PassRegistry&);
8081 void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
8182 void initializeBranchRelaxationPass(PassRegistry&);
0 //===-- CoalesceBranches.cpp - Coalesce blocks with the same condition ---===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Coalesce basic blocks guarded by the same branch condition into a single
11 /// basic block.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/ADT/BitVector.h"
16 #include "llvm/ADT/Statistic.h"
17 #include "llvm/CodeGen/MachineDominators.h"
18 #include "llvm/CodeGen/MachineFunctionPass.h"
19 #include "llvm/CodeGen/MachinePostDominators.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/Passes.h"
22 #include "llvm/Support/Debug.h"
23 #include "llvm/Target/TargetFrameLowering.h"
24 #include "llvm/Target/TargetInstrInfo.h"
25 #include "llvm/Target/TargetSubtargetInfo.h"
26
27 using namespace llvm;
28
29 #define DEBUG_TYPE "coal-branch"
30
31 static cl::opt
32 EnableBranchCoalescing("enable-branch-coalesce", cl::Hidden,
33 cl::desc("enable coalescing of duplicate branches"));
34
35 STATISTIC(NumBlocksCoalesced, "Number of blocks coalesced");
36 STATISTIC(NumPHINotMoved, "Number of PHI Nodes that cannot be merged");
37 STATISTIC(NumBlocksNotCoalesced, "Number of blocks not coalesced");
38
39 //===----------------------------------------------------------------------===//
40 // BranchCoalescing
41 //===----------------------------------------------------------------------===//
42 ///
43 /// Improve scheduling by coalescing branches that depend on the same condition.
44 /// This pass looks for blocks that are guarded by the same branch condition
45 /// and attempts to merge the blocks together. Such opportunities arise from
46 /// the expansion of select statements in the IR.
47 ///
48 /// For example, consider the following LLVM IR:
49 ///
50 /// %test = icmp eq i32 %x 0
51 /// %tmp1 = select i1 %test, double %a, double 2.000000e-03
52 /// %tmp2 = select i1 %test, double %b, double 5.000000e-03
53 ///
54 /// This IR expands to the following machine code on PowerPC:
55 ///
56 /// BB#0: derived from LLVM BB %entry
57 /// Live Ins: %F1 %F3 %X6
58 ///
59 /// %vreg0 = COPY %F1; F8RC:%vreg0
60 /// %vreg5 = CMPLWI %vreg4, 0; CRRC:%vreg5 GPRC:%vreg4
61 /// %vreg8 = LXSDX %ZERO8, %vreg7, %RM;
62 /// mem:LD8[ConstantPool] F8RC:%vreg8 G8RC:%vreg7
63 /// BCC 76, %vreg5, ; CRRC:%vreg5
64 /// Successors according to CFG: BB#1(?%) BB#2(?%)
65 ///
66 /// BB#1: derived from LLVM BB %entry
67 /// Predecessors according to CFG: BB#0
68 /// Successors according to CFG: BB#2(?%)
69 ///
70 /// BB#2: derived from LLVM BB %entry
71 /// Predecessors according to CFG: BB#0 BB#1
72 /// %vreg9 = PHI %vreg8, , %vreg0, ;
73 /// F8RC:%vreg9,%vreg8,%vreg0
74 ///
75 /// BCC 76, %vreg5, ; CRRC:%vreg5
76 /// Successors according to CFG: BB#3(?%) BB#4(?%)
77 ///
78 /// BB#3: derived from LLVM BB %entry
79 /// Predecessors according to CFG: BB#2
80 /// Successors according to CFG: BB#4(?%)
81 ///
82 /// BB#4: derived from LLVM BB %entry
83 /// Predecessors according to CFG: BB#2 BB#3
84 /// %vreg13 = PHI %vreg12, , %vreg2, ;
85 /// F8RC:%vreg13,%vreg12,%vreg2
86 ///
87 /// BLR8 %LR8, %RM, %F1
88 ///
89 /// When this pattern is detected, branch coalescing will try to collapse
90 /// it by moving code in BB#2 to BB#0 and/or BB#4 and removing BB#3.
91 ///
92 /// If all conditions are meet, IR should collapse to:
93 ///
94 /// BB#0: derived from LLVM BB %entry
95 /// Live Ins: %F1 %F3 %X6
96 ///
97 /// %vreg0 = COPY %F1; F8RC:%vreg0
98 /// %vreg5 = CMPLWI %vreg4, 0; CRRC:%vreg5 GPRC:%vreg4
99 /// %vreg8 = LXSDX %ZERO8, %vreg7, %RM;
100 /// mem:LD8[ConstantPool] F8RC:%vreg8 G8RC:%vreg7
101 ///
102 /// BCC 76, %vreg5, ; CRRC:%vreg5
103 /// Successors according to CFG: BB#1(0x2aaaaaaa / 0x80000000 = 33.33%)
104 /// BB#4(0x55555554 / 0x80000000 = 66.67%)
105 ///
106 /// BB#1: derived from LLVM BB %entry
107 /// Predecessors according to CFG: BB#0
108 /// Successors according to CFG: BB#4(0x40000000 / 0x80000000 = 50.00%)
109 ///
110 /// BB#4: derived from LLVM BB %entry
111 /// Predecessors according to CFG: BB#0 BB#1
112 /// %vreg9 = PHI %vreg8, , %vreg0, ;
113 /// F8RC:%vreg9,%vreg8,%vreg0
114 /// %vreg13 = PHI %vreg12, , %vreg2, ;
115 /// F8RC:%vreg13,%vreg12,%vreg2
116 ///
117 /// BLR8 %LR8, %RM, %F1
118 ///
119 /// Branch Coalescing does not split blocks, it moves everything in the same
120 /// direction ensuring it does not break use/definition semantics.
121 ///
122 /// PHI nodes and its corresponding use instructions are moved to its successor
123 /// block if there are no uses within the successor block PHI nodes. PHI
124 /// node ordering cannot be assumed.
125 ///
126 /// Non-PHI can be moved up to the predecessor basic block or down to the
127 /// successor basic block following any PHI instructions. Whether it moves
128 /// up or down depends on whether the register(s) defined in the instructions
129 /// are used in current block or in any PHI instructions at the beginning of
130 /// the successor block.
131
132 namespace {
133
134 class BranchCoalescing : public MachineFunctionPass {
135 struct CoalescingCandidateInfo {
136 MachineBasicBlock *BranchBlock; //< Block containing the branch
137 MachineBasicBlock *BranchTargetBlock; //< Block branched to
138 MachineBasicBlock *FallThroughBlock; //< Fall-through if branch not taken
139 SmallVector Cond;
140 bool MustMoveDown;
141 bool MustMoveUp;
142
143 CoalescingCandidateInfo();
144 void clear();
145 };
146
147 MachineDominatorTree *MDT;
148 MachinePostDominatorTree *MPDT;
149 const TargetInstrInfo *TII;
150 MachineRegisterInfo *MRI;
151
152 void initialize(MachineFunction &F);
153 bool canCoalesceBranch(CoalescingCandidateInfo &Cand);
154 bool identicalOperands(ArrayRef OperandList1,
155 ArrayRef OperandList2) const;
156 bool validateCandidates(CoalescingCandidateInfo &SourceRegion,
157 CoalescingCandidateInfo &TargetRegion) const;
158
159 static bool isBranchCoalescingEnabled() {
160 return EnableBranchCoalescing == cl::BOU_TRUE;
161 }
162
163 public:
164 static char ID;
165
166 BranchCoalescing() : MachineFunctionPass(ID) {
167 initializeBranchCoalescingPass(*PassRegistry::getPassRegistry());
168 }
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
171 AU.addRequired();
172 AU.addRequired();
173 MachineFunctionPass::getAnalysisUsage(AU);
174 }
175
176 StringRef getPassName() const override { return "Branch Coalescing"; }
177
178 bool mergeCandidates(CoalescingCandidateInfo &SourceRegion,
179 CoalescingCandidateInfo &TargetRegion);
180 bool canMoveToBeginning(const MachineInstr &MI,
181 const MachineBasicBlock &MBB) const;
182 bool canMoveToEnd(const MachineInstr &MI,
183 const MachineBasicBlock &MBB) const;
184 bool canMerge(CoalescingCandidateInfo &SourceRegion,
185 CoalescingCandidateInfo &TargetRegion) const;
186 void moveAndUpdatePHIs(MachineBasicBlock *SourceRegionMBB,
187 MachineBasicBlock *TargetRegionMBB);
188 bool runOnMachineFunction(MachineFunction &MF) override;
189 };
190 } // End anonymous namespace.
191
192 char BranchCoalescing::ID = 0;
193 char &llvm::BranchCoalescingID = BranchCoalescing::ID;
194
195 INITIALIZE_PASS_BEGIN(BranchCoalescing, "branch-coalescing",
196 "Branch Coalescing", false, false)
197 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
198 INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
199 INITIALIZE_PASS_END(BranchCoalescing, "branch-coalescing", "Branch Coalescing",
200 false, false)
201
202 BranchCoalescing::CoalescingCandidateInfo::CoalescingCandidateInfo()
203 : BranchBlock(nullptr), BranchTargetBlock(nullptr),
204 FallThroughBlock(nullptr), MustMoveDown(false), MustMoveUp(false) {}
205
206 void BranchCoalescing::CoalescingCandidateInfo::clear() {
207 BranchBlock = nullptr;
208 BranchTargetBlock = nullptr;
209 FallThroughBlock = nullptr;
210 Cond.clear();
211 MustMoveDown = false;
212 MustMoveUp = false;
213 }
214
215 void BranchCoalescing::initialize(MachineFunction &MF) {
216 MDT = &getAnalysis();
217 MPDT = &getAnalysis();
218 TII = MF.getSubtarget().getInstrInfo();
219 MRI = &MF.getRegInfo();
220 }
221
222 ///
223 /// Analyze the branch statement to determine if it can be coalesced. This
224 /// method analyses the branch statement for the given candidate to determine
225 /// if it can be coalesced. If the branch can be coalesced, then the
226 /// BranchTargetBlock and the FallThroughBlock are recorded in the specified
227 /// Candidate.
228 ///
229 ///\param[in,out] Cand The coalescing candidate to analyze
230 ///\return true if and only if the branch can be coalesced, false otherwise
231 ///
232 bool BranchCoalescing::canCoalesceBranch(CoalescingCandidateInfo &Cand) {
233 DEBUG(dbgs() << "Determine if branch block " << Cand.BranchBlock->getNumber()
234 << " can be coalesced:");
235 MachineBasicBlock *FalseMBB = nullptr;
236
237 if (TII->analyzeBranch(*Cand.BranchBlock, Cand.BranchTargetBlock, FalseMBB,
238 Cand.Cond)) {
239 DEBUG(dbgs() << "TII unable to Analyze Branch - skip\n");
240 return false;
241 }
242
243 for (auto &I : Cand.BranchBlock->terminators()) {
244 DEBUG(dbgs() << "Looking at terminator : " << I << "\n");
245 if (!I.isBranch())
246 continue;
247
248 if (I.getNumOperands() != I.getNumExplicitOperands()) {
249 DEBUG(dbgs() << "Terminator contains implicit operands - skip : " << I
250 << "\n");
251 return false;
252 }
253 }
254
255 if (Cand.BranchBlock->isEHPad() || Cand.BranchBlock->hasEHPadSuccessor()) {
256 DEBUG(dbgs() << "EH Pad - skip\n");
257 return false;
258 }
259
260 // For now only consider triangles (i.e, BranchTargetBlock is set,
261 // FalseMBB is null, and BranchTargetBlock is a successor to BranchBlock)
262 if (!Cand.BranchTargetBlock || (Cand.BranchTargetBlock && FalseMBB)
263 || !Cand.BranchBlock->isSuccessor(Cand.BranchTargetBlock)) {
264 DEBUG(dbgs() << "Does not form a triangle - skip\n");
265 return false;
266 }
267
268 // Ensure there are only two successors
269 if (Cand.BranchBlock->succ_size() != 2) {
270 DEBUG(dbgs() << "Does not have 2 successors - skip\n");
271 return false;
272 }
273
274 // Sanity check - the block must be able to fall through
275 assert(Cand.BranchBlock->canFallThrough() &&
276 "Expecting the block to fall through!");
277
278 // We have already ensured there are exactly two successors to
279 // BranchBlock and that BranchTargetBlock is a successor to BranchBlock.
280 // Ensure the single fall though block is empty.
281 MachineBasicBlock *Succ =
282 (*Cand.BranchBlock->succ_begin() == Cand.BranchTargetBlock)
283 ? *Cand.BranchBlock->succ_rbegin()
284 : *Cand.BranchBlock->succ_begin();
285
286 assert(Succ && "Expecting a valid fall-through block\n");
287
288 if (!Succ->empty()) {
289 DEBUG(dbgs() << "Fall-through block contains code -- skip\n");
290 return false;
291 }
292
293 if (!Succ->isSuccessor(Cand.BranchTargetBlock)) {
294 DEBUG(dbgs()
295 << "Successor of fall through block is not branch taken block\n");
296 return false;
297 }
298
299 Cand.FallThroughBlock = Succ;
300 DEBUG(dbgs() << "Valid Candidate\n");
301 return true;
302 }
303
304 ///
305 /// Determine if the two operand lists are identical
306 ///
307 /// \param[in] OpList1 operand list
308 /// \param[in] OpList2 operand list
309 /// \return true if and only if the operands lists are identical
310 ///
311 bool BranchCoalescing::identicalOperands(
312 ArrayRef OpList1, ArrayRef OpList2) const {
313
314 if (OpList1.size() != OpList2.size()) {
315 DEBUG(dbgs() << "Operand list is different size\n");
316 return false;
317 }
318
319 for (unsigned i = 0; i < OpList1.size(); ++i) {
320 const MachineOperand &Op1 = OpList1[i];
321 const MachineOperand &Op2 = OpList2[i];
322
323 DEBUG(dbgs() << "Op1: " << Op1 << "\n"
324 << "Op2: " << Op2 << "\n");
325
326 if (Op1.isIdenticalTo(Op2)) {
327 DEBUG(dbgs() << "Op1 and Op2 are identical!\n");
328 continue;
329 }
330
331 // If the operands are not identical, but are registers, check to see if the
332 // definition of the register produces the same value. If they produce the
333 // same value, consider them to be identical.
334 if (Op1.isReg() && Op2.isReg() &&
335 TargetRegisterInfo::isVirtualRegister(Op1.getReg()) &&
336 TargetRegisterInfo::isVirtualRegister(Op2.getReg())) {
337 MachineInstr *Op1Def = MRI->getVRegDef(Op1.getReg());
338 MachineInstr *Op2Def = MRI->getVRegDef(Op2.getReg());
339 if (TII->produceSameValue(*Op1Def, *Op2Def, MRI)) {
340 DEBUG(dbgs() << "Op1Def: " << *Op1Def << " and " << *Op2Def
341 << " produce the same value!\n");
342 } else {
343 DEBUG(dbgs() << "Operands produce different values\n");
344 return false;
345 }
346 } else {
347 DEBUG(dbgs() << "The operands are not provably identical.\n");
348 return false;
349 }
350 }
351 return true;
352 }
353
354 ///
355 /// Moves ALL PHI instructions in SourceMBB to beginning of TargetMBB
356 /// and update them to refer to the new block. PHI node ordering
357 /// cannot be assumed so it does not matter where the PHI instructions
358 /// are moved to in TargetMBB.
359 ///
360 /// \param[in] SourceMBB block to move PHI instructions from
361 /// \param[in] TargetMBB block to move PHI instructions to
362 ///
363 void BranchCoalescing::moveAndUpdatePHIs(MachineBasicBlock *SourceMBB,
364 MachineBasicBlock *TargetMBB) {
365
366 MachineBasicBlock::iterator MI = SourceMBB->begin();
367 MachineBasicBlock::iterator ME = SourceMBB->getFirstNonPHI();
368
369 if (MI == ME) {
370 DEBUG(dbgs() << "SourceMBB contains no PHI instructions.\n");
371 return;
372 }
373
374 // Update all PHI instructions in SourceMBB and move to top of TargetMBB
375 for (MachineBasicBlock::iterator Iter = MI; Iter != ME; Iter++) {
376 MachineInstr &PHIInst = *Iter;
377 for (unsigned i = 2, e = PHIInst.getNumOperands() + 1; i != e; i += 2) {
378 MachineOperand &MO = PHIInst.getOperand(i);
379 if (MO.getMBB() == SourceMBB)
380 MO.setMBB(TargetMBB);
381 }
382 }
383 TargetMBB->splice(TargetMBB->begin(), SourceMBB, MI, ME);
384 }
385
386 ///
387 /// This function checks if MI can be moved to the beginning of the TargetMBB
388 /// following PHI instructions. A MI instruction can be moved to beginning of
389 /// the TargetMBB if there are no uses of it within the TargetMBB PHI nodes.
390 ///
391 /// \param[in] MI the machine instruction to move.
392 /// \param[in] MBB the machine basic block to move to
393 /// \return true if it is safe to move MI to beginning of TargetMBB,
394 /// false otherwise.
395 ///
396 bool BranchCoalescing::canMoveToBeginning(const MachineInstr &MI,
397 const MachineBasicBlock &TargetMBB
398 ) const {
399
400 DEBUG(dbgs() << "Checking if " << MI << " can move to beginning of "
401 << TargetMBB.getNumber() << "\n");
402
403 for (auto &Def : MI.defs()) { // Looking at Def
404 for (auto &Use : MRI->use_instructions(Def.getReg())) {
405 if (Use.isPHI() && Use.getParent() == &TargetMBB) {
406 DEBUG(dbgs() << " *** used in a PHI -- cannot move ***\n");
407 return false;
408 }
409 }
410 }
411
412 DEBUG(dbgs() << " Safe to move to the beginning.\n");
413 return true;
414 }
415
416 ///
417 /// This function checks if MI can be moved to the end of the TargetMBB,
418 /// immediately before the first terminator. A MI instruction can be moved
419 /// to then end of the TargetMBB if no PHI node defines what MI uses within
420 /// it's own MBB.
421 ///
422 /// \param[in] MI the machine instruction to move.
423 /// \param[in] MBB the machine basic block to move to
424 /// \return true if it is safe to move MI to end of TargetMBB,
425 /// false otherwise.
426 ///
427 bool BranchCoalescing::canMoveToEnd(const MachineInstr &MI,
428 const MachineBasicBlock &TargetMBB
429 ) const {
430
431 DEBUG(dbgs() << "Checking if " << MI << " can move to end of "
432 << TargetMBB.getNumber() << "\n");
433
434 for (auto &Use : MI.uses()) {
435 if (Use.isReg() && TargetRegisterInfo::isVirtualRegister(Use.getReg())) {
436 MachineInstr *DefInst = MRI->getVRegDef(Use.getReg());
437 if (DefInst->isPHI() && DefInst->getParent() == MI.getParent()) {
438 DEBUG(dbgs() << " *** Cannot move this instruction ***\n");
439 return false;
440 } else {
441 DEBUG(dbgs() << " *** def is in another block -- safe to move!\n");
442 }
443 }
444 }
445
446 DEBUG(dbgs() << " Safe to move to the end.\n");
447 return true;
448 }
449
450 ///
451 /// This method checks to ensure the two coalescing candidates follows the
452 /// expected pattern required for coalescing.
453 ///
454 /// \param[in] SourceRegion The candidate to move statements from
455 /// \param[in] TargetRegion The candidate to move statements to
456 /// \return true if all instructions in SourceRegion.BranchBlock can be merged
457 /// into a block in TargetRegion; false otherwise.
458 ///
459 bool BranchCoalescing::validateCandidates(
460 CoalescingCandidateInfo &SourceRegion,
461 CoalescingCandidateInfo &TargetRegion) const {
462
463 if (TargetRegion.BranchTargetBlock != SourceRegion.BranchBlock)
464 llvm_unreachable("Expecting SourceRegion to immediately follow TargetRegion");
465 else if (!MDT->dominates(TargetRegion.BranchBlock, SourceRegion.BranchBlock))
466 llvm_unreachable("Expecting TargetRegion to dominate SourceRegion");
467 else if (!MPDT->dominates(SourceRegion.BranchBlock, TargetRegion.BranchBlock))
468 llvm_unreachable("Expecting SourceRegion to post-dominate TargetRegion");
469 else if (!TargetRegion.FallThroughBlock->empty() ||
470 !SourceRegion.FallThroughBlock->empty())
471 llvm_unreachable("Expecting fall-through blocks to be empty");
472
473 return true;
474 }
475
476 ///
477 /// This method determines whether the two coalescing candidates can be merged.
478 /// In order to be merged, all instructions must be able to
479 /// 1. Move to the beginning of the SourceRegion.BranchTargetBlock;
480 /// 2. Move to the end of the TargetRegion.BranchBlock.
481 /// Merging involves moving the instructions in the
482 /// TargetRegion.BranchTargetBlock (also SourceRegion.BranchBlock).
483 ///
484 /// This function first try to move instructions from the
485 /// TargetRegion.BranchTargetBlock down, to the beginning of the
486 /// SourceRegion.BranchTargetBlock. This is not possible if any register defined
487 /// in TargetRegion.BranchTargetBlock is used in a PHI node in the
488 /// SourceRegion.BranchTargetBlock. In this case, check whether the statement
489 /// can be moved up, to the end of the TargetRegion.BranchBlock (immediately
490 /// before the branch statement). If it cannot move, then these blocks cannot
491 /// be merged.
492 ///
493 /// Note that there is no analysis for moving instructions past the fall-through
494 /// blocks because they are confirmed to be empty. An assert is thrown if they
495 /// are not.
496 ///
497 /// \param[in] SourceRegion The candidate to move statements from
498 /// \param[in] TargetRegion The candidate to move statements to
499 /// \return true if all instructions in SourceRegion.BranchBlock can be merged
500 /// into a block in TargetRegion, false otherwise.
501 ///
502 bool BranchCoalescing::canMerge(CoalescingCandidateInfo &SourceRegion,
503 CoalescingCandidateInfo &TargetRegion) const {
504 if (!validateCandidates(SourceRegion, TargetRegion))
505 return false;
506
507 // Walk through PHI nodes first and see if they force the merge into the
508 // SourceRegion.BranchTargetBlock.
509 for (MachineBasicBlock::iterator
510 I = SourceRegion.BranchBlock->instr_begin(),
511 E = SourceRegion.BranchBlock->getFirstNonPHI();
512 I != E; ++I) {
513 for (auto &Def : I->defs())
514 for (auto &Use : MRI->use_instructions(Def.getReg())) {
515 if (Use.isPHI() && Use.getParent() == SourceRegion.BranchTargetBlock) {
516 DEBUG(dbgs() << "PHI " << *I << " defines register used in another "
517 "PHI within branch target block -- can't merge\n");
518 NumPHINotMoved++;
519 return false;
520 }
521 if (Use.getParent() == SourceRegion.BranchBlock) {
522 DEBUG(dbgs() << "PHI " << *I
523 << " defines register used in this "
524 "block -- all must move down\n");
525 SourceRegion.MustMoveDown = true;
526 }
527 }
528 }
529
530 // Walk through the MI to see if they should be merged into
531 // TargetRegion.BranchBlock (up) or SourceRegion.BranchTargetBlock (down)
532 for (MachineBasicBlock::iterator
533 I = SourceRegion.BranchBlock->getFirstNonPHI(),
534 E = SourceRegion.BranchBlock->end();
535 I != E; ++I) {
536 if (!canMoveToBeginning(*I, *SourceRegion.BranchTargetBlock)) {
537 DEBUG(dbgs() << "Instruction " << *I
538 << " cannot move down - must move up!\n");
539 SourceRegion.MustMoveUp = true;
540 }
541 if (!canMoveToEnd(*I, *TargetRegion.BranchBlock)) {
542 DEBUG(dbgs() << "Instruction " << *I
543 << " cannot move up - must move down!\n");
544 SourceRegion.MustMoveDown = true;
545 }
546 }
547
548 return (SourceRegion.MustMoveUp && SourceRegion.MustMoveDown) ? false : true;
549 }
550
551 /// Merge the instructions from SourceRegion.BranchBlock,
552 /// SourceRegion.BranchTargetBlock, and SourceRegion.FallThroughBlock into
553 /// TargetRegion.BranchBlock, TargetRegion.BranchTargetBlock and
554 /// TargetRegion.FallThroughBlock respectively.
555 ///
556 /// The successors for blocks in TargetRegion will be updated to use the
557 /// successors from blocks in SourceRegion. Finally, the blocks in SourceRegion
558 /// will be removed from the function.
559 ///
560 /// A region consists of a BranchBlock, a FallThroughBlock, and a
561 /// BranchTargetBlock. Branch coalesce works on patterns where the
562 /// TargetRegion's BranchTargetBlock must also be the SourceRegions's
563 /// BranchBlock.
564 ///
565 /// Before mergeCandidates:
566 ///
567 /// +---------------------------+
568 /// | TargetRegion.BranchBlock |
569 /// +---------------------------+
570 /// / |
571 /// / +--------------------------------+
572 /// | | TargetRegion.FallThroughBlock |
573 /// \ +--------------------------------+
574 /// \ |
575 /// +----------------------------------+
576 /// | TargetRegion.BranchTargetBlock |
577 /// | SourceRegion.BranchBlock |
578 /// +----------------------------------+
579 /// / |
580 /// / +--------------------------------+
581 /// | | SourceRegion.FallThroughBlock |
582 /// \ +--------------------------------+
583 /// \ |
584 /// +----------------------------------+
585 /// | SourceRegion.BranchTargetBlock |
586 /// +----------------------------------+
587 ///
588 /// After mergeCandidates:
589 ///
590 /// +-----------------------------+
591 /// | TargetRegion.BranchBlock |
592 /// | SourceRegion.BranchBlock |
593 /// +-----------------------------+
594 /// / |
595 /// / +---------------------------------+
596 /// | | TargetRegion.FallThroughBlock |
597 /// | | SourceRegion.FallThroughBlock |
598 /// \ +---------------------------------+
599 /// \ |
600 /// +----------------------------------+
601 /// | SourceRegion.BranchTargetBlock |
602 /// +----------------------------------+
603 ///
604 /// \param[in] SourceRegion The candidate to move blocks from
605 /// \param[in] TargetRegion The candidate to move blocks to
606 ///
607 bool BranchCoalescing::mergeCandidates(CoalescingCandidateInfo &SourceRegion,
608 CoalescingCandidateInfo &TargetRegion) {
609
610 if (SourceRegion.MustMoveUp && SourceRegion.MustMoveDown) {
611 llvm_unreachable("Cannot have both MustMoveDown and MustMoveUp set!");
612 return false;
613 }
614
615 if (!validateCandidates(SourceRegion, TargetRegion))
616 return false;
617
618 // Start the merging process by first handling the BranchBlock.
619 // Move any PHIs in SourceRegion.BranchBlock down to the branch-taken block
620 moveAndUpdatePHIs(SourceRegion.BranchBlock, SourceRegion.BranchTargetBlock);
621
622 // Move remaining instructions in SourceRegion.BranchBlock into
623 // TargetRegion.BranchBlock
624 MachineBasicBlock::iterator firstInstr =
625 SourceRegion.BranchBlock->getFirstNonPHI();
626 MachineBasicBlock::iterator lastInstr =
627 SourceRegion.BranchBlock->getFirstTerminator();
628
629 MachineBasicBlock *Source = SourceRegion.MustMoveDown
630 ? SourceRegion.BranchTargetBlock
631 : TargetRegion.BranchBlock;
632
633 MachineBasicBlock::iterator Target =
634 SourceRegion.MustMoveDown
635 ? SourceRegion.BranchTargetBlock->getFirstNonPHI()
636 : TargetRegion.BranchBlock->getFirstTerminator();
637
638 Source->splice(Target, SourceRegion.BranchBlock, firstInstr, lastInstr);
639
640 // Once PHI and instructions have been moved we need to clean up the
641 // control flow.
642
643 // Remove SourceRegion.FallThroughBlock before transferring successors of
644 // SourceRegion.BranchBlock to TargetRegion.BranchBlock.
645 SourceRegion.BranchBlock->removeSuccessor(SourceRegion.FallThroughBlock);
646 TargetRegion.BranchBlock->transferSuccessorsAndUpdatePHIs(
647 SourceRegion.BranchBlock);
648 // Update branch in TargetRegion.BranchBlock to jump to
649 // SourceRegion.BranchTargetBlock
650 // In this case, TargetRegion.BranchTargetBlock == SourceRegion.BranchBlock.
651 TargetRegion.BranchBlock->ReplaceUsesOfBlockWith(
652 SourceRegion.BranchBlock, SourceRegion.BranchTargetBlock);
653 // Remove the branch statement(s) in SourceRegion.BranchBlock
654 MachineBasicBlock::iterator I =
655 SourceRegion.BranchBlock->terminators().begin();
656 while (I != SourceRegion.BranchBlock->terminators().end()) {
657 MachineInstr &CurrInst = *I;
658 ++I;
659 if (CurrInst.isBranch())
660 CurrInst.eraseFromParent();
661 }
662
663 // Fall-through block should be empty since this is part of the condition
664 // to coalesce the branches.
665 assert(TargetRegion.FallThroughBlock->empty() &&
666 "FallThroughBlocks should be empty!");
667
668 // Transfer successor information and move PHIs down to the
669 // branch-taken block.
670 TargetRegion.FallThroughBlock->transferSuccessorsAndUpdatePHIs(
671 SourceRegion.FallThroughBlock);
672 TargetRegion.FallThroughBlock->removeSuccessor(SourceRegion.BranchBlock);
673
674 // Remove the blocks from the function.
675 assert(SourceRegion.BranchBlock->empty() &&
676 "Expecting branch block to be empty!");
677 SourceRegion.BranchBlock->eraseFromParent();
678
679 assert(SourceRegion.FallThroughBlock->empty() &&
680 "Expecting fall-through block to be empty!\n");
681 SourceRegion.FallThroughBlock->eraseFromParent();
682
683 NumBlocksCoalesced++;
684 return true;
685 }
686
687 bool BranchCoalescing::runOnMachineFunction(MachineFunction &MF) {
688
689 if (skipFunction(*MF.getFunction()) || MF.empty() ||
690 !isBranchCoalescingEnabled())
691 return false;
692
693 bool didSomething = false;
694
695 DEBUG(dbgs() << "******** Branch Coalescing ********\n");
696 initialize(MF);
697
698 DEBUG(dbgs() << "Function: "; MF.dump(); dbgs() << "\n");
699
700 CoalescingCandidateInfo Cand1, Cand2;
701 // Walk over blocks and find candidates to merge
702 // Continue trying to merge with the first candidate found, as long as merging
703 // is successfull.
704 for (MachineBasicBlock &MBB : MF) {
705 bool MergedCandidates = false;
706 do {
707 MergedCandidates = false;
708 Cand1.clear();
709 Cand2.clear();
710
711 Cand1.BranchBlock = &MBB;
712
713 // If unable to coalesce the branch, then continue to next block
714 if (!canCoalesceBranch(Cand1))
715 break;
716
717 Cand2.BranchBlock = Cand1.BranchTargetBlock;
718 if (!canCoalesceBranch(Cand2))
719 break;
720
721 // Sanity check
722 // The branch-taken block of the second candidate should post-dominate the
723 // first candidate
724 assert(MPDT->dominates(Cand2.BranchTargetBlock, Cand1.BranchBlock) &&
725 "Branch-taken block should post-dominate first candidate");
726
727 if (!identicalOperands(Cand1.Cond, Cand2.Cond)) {
728 DEBUG(dbgs() << "Blocks " << Cand1.BranchBlock->getNumber() << " and "
729 << Cand2.BranchBlock->getNumber()
730 << " have different branches\n");
731 break;
732 }
733 if (!canMerge(Cand2, Cand1)) {
734 DEBUG(dbgs() << "Cannot merge blocks " << Cand1.BranchBlock->getNumber()
735 << " and " << Cand2.BranchBlock->getNumber() << "\n");
736 NumBlocksNotCoalesced++;
737 continue;
738 }
739 DEBUG(dbgs() << "Merging blocks " << Cand1.BranchBlock->getNumber()
740 << " and " << Cand1.BranchTargetBlock->getNumber() << "\n");
741 MergedCandidates = mergeCandidates(Cand2, Cand1);
742 if (MergedCandidates)
743 didSomething = true;
744
745 DEBUG(dbgs() << "Function after merging: "; MF.dump(); dbgs() << "\n");
746 } while (MergedCandidates);
747 }
748
749 #ifndef NDEBUG
750 // Verify MF is still valid after branch coalescing
751 if (didSomething)
752 MF.verify(nullptr, "Error in code produced by branch coalescing");
753 #endif // NDEBUG
754
755 DEBUG(dbgs() << "Finished Branch Coalescing\n");
756 return didSomething;
757 }
33 Analysis.cpp
44 AtomicExpandPass.cpp
55 BasicTargetTransformInfo.cpp
6 BranchCoalescing.cpp
67 BranchFolding.cpp
78 BranchRelaxation.cpp
89 BuiltinGCs.cpp
2020 /// initializeCodeGen - Initialize all passes linked into the CodeGen library.
2121 void llvm::initializeCodeGen(PassRegistry &Registry) {
2222 initializeAtomicExpandPass(Registry);
23 initializeBranchCoalescingPass(Registry);
2324 initializeBranchFolderPassPass(Registry);
2425 initializeBranchRelaxationPass(Registry);
2526 initializeCodeGenPreparePass(Registry);
706706
707707 addPass(&MachineLICMID, false);
708708 addPass(&MachineCSEID, false);
709
710 // Coalesce basic blocks with the same branch condition
711 addPass(&BranchCoalescingID);
712
709713 addPass(&MachineSinkingID);
710714
711715 addPass(&PeepholeOptimizerID);
0 ; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -enable-branch-coalesce=true < %s | FileCheck %s
1 ; RUN: llc -mcpu=pwr8 -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -enable-branch-coalesce=true < %s | FileCheck %s
2
3 ; Function Attrs: nounwind
4 define double @testBranchCoal(double %a, double %b, double %c, i32 %x) {
5 entry:
6 %test = icmp eq i32 %x, 0
7 %tmp1 = select i1 %test, double %a, double 2.000000e-03
8 %tmp2 = select i1 %test, double %b, double 0.000000e+00
9 %tmp3 = select i1 %test, double %c, double 5.000000e-03
10
11 %res1 = fadd double %tmp1, %tmp2
12 %result = fadd double %res1, %tmp3
13 ret double %result
14
15 ; CHECK-LABEL: @testBranchCoal
16 ; CHECK: cmplwi [[CMPR:[0-7]+]], 6, 0
17 ; CHECK: beq [[CMPR]], .LBB[[LAB1:[0-9_]+]]
18 ; CHECK-DAG: addis [[LD1REG:[0-9]+]], 2, .LCPI0_0@toc@ha
19 ; CHECK-DAG: addis [[LD2REG:[0-9]+]], 2, .LCPI0_1@toc@ha
20 ; CHECK-DAG: xxlxor 2, 2, 2
21 ; CHECK-NOT: beq
22 ; CHECK-DAG: addi [[LD1BASE:[0-9]+]], [[LD1REG]]
23 ; CHECK-DAG: addi [[LD2BASE:[0-9]+]], [[LD2REG]]
24 ; CHECK-DAG: lxsdx 1, 0, [[LD1BASE]]
25 ; CHECK-DAG: lxsdx 3, 0, [[LD2BASE]]
26 ; CHECK: .LBB[[LAB1]]
27 ; CHECK: xsadddp 0, 1, 2
28 ; CHECK: xsadddp 1, 0, 3
29 ; CHECK: blr
30 }