llvm.org GIT mirror llvm / 7cc2b07
Introduce MachineBranchProbabilityInfo class, which has similar API to BranchProbabilityInfo (expect setEdgeWeight which is not available here). Branch Weights are kept in MachineBasicBlocks. To turn off this analysis set -use-mbpi=false. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@133184 91177308-0d34-0410-b5e6-96231b3b80d8 Jakub Staszak 9 years ago
12 changed file(s) with 358 addition(s) and 29 deletion(s). Raw diff Collapse all Expand all
2424 class BranchProbabilityInfo : public FunctionPass {
2525
2626 // Default weight value. Used when we don't have information about the edge.
27 // TODO: DEFAULT_WEIGHT makes sense during static predication, when none of
28 // the successors have a weight yet. But it doesn't make sense when providing
29 // weight to an edge that may have siblings with non-zero weights. This can
30 // be handled various ways, but it's probably fine for an edge with unknown
31 // weight to just "inherit" the non-zero weight of an adjacent successor.
2732 static const uint32_t DEFAULT_WEIGHT = 16;
2833
2934 typedef std::pair Edge;
2323 #ifndef NDEBUG
2424 #include "llvm/ADT/SmallSet.h"
2525 #endif
26 #include "llvm/Analysis/BranchProbabilityInfo.h"
2627 #include "llvm/CodeGen/ValueTypes.h"
2728 #include "llvm/CodeGen/ISDOpcodes.h"
2829 #include "llvm/CodeGen/MachineBasicBlock.h"
5657 const Function *Fn;
5758 MachineFunction *MF;
5859 MachineRegisterInfo *RegInfo;
59
60 BranchProbabilityInfo *BPI;
6061 /// CanLowerReturn - true iff the function's return value can be lowered to
6162 /// registers.
6263 bool CanLowerReturn;
1515
1616 #include "llvm/CodeGen/MachineInstr.h"
1717 #include "llvm/ADT/GraphTraits.h"
18 #include "llvm/Support/DataTypes.h"
1819 #include
1920
2021 namespace llvm {
2627 class SlotIndexes;
2728 class StringRef;
2829 class raw_ostream;
30 class MachineBranchProbabilityInfo;
2931
3032 template <>
3133 struct ilist_traits : public ilist_default_traits {
6264 const BasicBlock *BB;
6365 int Number;
6466 MachineFunction *xParent;
65
67
6668 /// Predecessors/Successors - Keep track of the predecessor / successor
6769 /// basicblocks.
6870 std::vector Predecessors;
6971 std::vector Successors;
72
73
74 /// Weights - Keep track of the weights to the successors. This vector
75 /// has the same order as Successors, or it is empty if we don't use it
76 /// (disable optimization).
77 std::vector Weights;
78 typedef std::vector::iterator weight_iterator;
7079
7180 /// LiveIns - Keep track of the physical registers that are livein of
7281 /// the basicblock.
243252 void updateTerminator();
244253
245254 // Machine-CFG mutators
246
255
247256 /// addSuccessor - Add succ as a successor of this MachineBasicBlock.
248 /// The Predecessors list of succ is automatically updated.
249 ///
250 void addSuccessor(MachineBasicBlock *succ);
257 /// The Predecessors list of succ is automatically updated. WEIGHT
258 /// parameter is stored in Weights list and it may be used by
259 /// MachineBranchProbabilityInfo analysis to calculate branch probability.
260 ///
261 void addSuccessor(MachineBasicBlock *succ, uint32_t weight = 0);
251262
252263 /// removeSuccessor - Remove successor from the successors list of this
253264 /// MachineBasicBlock. The Predecessors list of succ is automatically updated.
259270 /// updated. Return the iterator to the element after the one removed.
260271 ///
261272 succ_iterator removeSuccessor(succ_iterator I);
262
273
274 /// replaceSuccessor - Replace successor OLD with NEW and update weight info.
275 ///
276 void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New);
277
278
263279 /// transferSuccessors - Transfers all the successors from MBB to this
264280 /// machine basic block (i.e., copies all the successors fromMBB and
265281 /// remove all the successors from fromMBB).
395411 /// getSymbol - Return the MCSymbol for this basic block.
396412 ///
397413 MCSymbol *getSymbol() const;
398
399 private: // Methods used to maintain doubly linked list of blocks...
414
415
416 private:
417 /// getWeightIterator - Return weight iterator corresponding to the I
418 /// successor iterator.
419 weight_iterator getWeightIterator(succ_iterator I);
420
421 friend class MachineBranchProbabilityInfo;
422
423 /// getSuccWeight - Return weight of the edge from this block to MBB. This
424 /// method should NOT be called directly, but by using getEdgeWeight method
425 /// from MachineBranchProbabilityInfo class.
426 uint32_t getSuccWeight(MachineBasicBlock *succ);
427
428
429 // Methods used to maintain doubly linked list of blocks...
400430 friend struct ilist_traits;
401431
402432 // Machine-CFG mutators
0
1 //==- MachineBranchProbabilityInfo.h - Machine Branch Probability Analysis -==//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass is used to evaluate branch probabilties on machine basic blocks.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_CODEGEN_MACHINEBRANCHPROBABILITYINFO_H
15 #define LLVM_CODEGEN_MACHINEBRANCHPROBABILITYINFO_H
16
17 #include "llvm/Pass.h"
18 #include "llvm/Support/BranchProbability.h"
19 #include
20
21 namespace llvm {
22
23 class raw_ostream;
24
25 class MachineBranchProbabilityInfo : public ImmutablePass {
26
27 // Default weight value. Used when we don't have information about the edge.
28 // TODO: DEFAULT_WEIGHT makes sense during static predication, when none of
29 // the successors have a weight yet. But it doesn't make sense when providing
30 // weight to an edge that may have siblings with non-zero weights. This can
31 // be handled various ways, but it's probably fine for an edge with unknown
32 // weight to just "inherit" the non-zero weight of an adjacent successor.
33 static const uint32_t DEFAULT_WEIGHT = 16;
34
35 // Get sum of the block successors' weights.
36 uint32_t getSumForBlock(MachineBasicBlock *MBB) const;
37
38 public:
39 static char ID;
40
41 MachineBranchProbabilityInfo() : ImmutablePass(ID) {
42 PassRegistry &Registry = *PassRegistry::getPassRegistry();
43 initializeMachineBranchProbabilityInfoPass(Registry);
44 }
45
46 void getAnalysisUsage(AnalysisUsage &AU) const {
47 AU.setPreservesAll();
48 }
49
50 // Return edge weight. If we don't have any informations about it - return
51 // DEFAULT_WEIGHT.
52 uint32_t getEdgeWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst) const;
53
54 // A 'Hot' edge is an edge which probability is >= 80%.
55 bool isEdgeHot(MachineBasicBlock *Src, MachineBasicBlock *Dst) const;
56
57 // Return a hot successor for the block BB or null if there isn't one.
58 MachineBasicBlock *getHotSucc(MachineBasicBlock *MBB) const;
59
60 // Return a probability as a fraction between 0 (0% probability) and
61 // 1 (100% probability), however the value is never equal to 0, and can be 1
62 // only iff SRC block has only one successor.
63 BranchProbability getEdgeProbability(MachineBasicBlock *Src,
64 MachineBasicBlock *Dst) const;
65
66 // Print value between 0 (0% probability) and 1 (100% probability),
67 // however the value is never equal to 0, and can be 1 only iff SRC block
68 // has only one successor.
69 raw_ostream &printEdgeProbability(raw_ostream &OS, MachineBasicBlock *Src,
70 MachineBasicBlock *Dst) const;
71 };
72
73 }
74
75
76 #endif
143143 void initializeLowerInvokePass(PassRegistry&);
144144 void initializeLowerSetJmpPass(PassRegistry&);
145145 void initializeLowerSwitchPass(PassRegistry&);
146 void initializeMachineBranchProbabilityInfoPass(PassRegistry&);
146147 void initializeMachineCSEPass(PassRegistry&);
147148 void initializeMachineDominatorTreePass(PassRegistry&);
148149 void initializeMachineLICMPass(PassRegistry&);
347347 raw_ostream &
348348 BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS, BasicBlock *Src,
349349 BasicBlock *Dst) const {
350 BranchProbability Prob = getEdgeProbability(Src, Dst);
351
350
351 const BranchProbability Prob = getEdgeProbability(Src, Dst);
352352 OS << "edge " << Src->getNameStr() << " -> " << Dst->getNameStr()
353353 << " probability is " << Prob
354354 << (isEdgeHot(Src, Dst) ? " [HOT edge]\n" : "\n");
3232 LocalStackSlotAllocation.cpp
3333 LowerSubregs.cpp
3434 MachineBasicBlock.cpp
35 MachineBranchProbabilityInfo.cpp
3536 MachineCSE.cpp
3637 MachineDominators.cpp
3738 MachineFunction.cpp
338338 }
339339 }
340340
341 void MachineBasicBlock::addSuccessor(MachineBasicBlock *succ) {
342 Successors.push_back(succ);
343 succ->addPredecessor(this);
344 }
341 void MachineBasicBlock::addSuccessor(MachineBasicBlock *succ, uint32_t weight) {
342
343 // If we see non-zero value for the first time it means we actually use Weight
344 // list, so we fill all Weights with 0's.
345 if (weight != 0 && Weights.empty())
346 Weights.resize(Successors.size());
347
348 if (weight != 0 || !Weights.empty())
349 Weights.push_back(weight);
350
351 Successors.push_back(succ);
352 succ->addPredecessor(this);
353 }
345354
346355 void MachineBasicBlock::removeSuccessor(MachineBasicBlock *succ) {
347356 succ->removePredecessor(this);
348357 succ_iterator I = std::find(Successors.begin(), Successors.end(), succ);
349358 assert(I != Successors.end() && "Not a current successor!");
359
360 // If Weight list is empty it means we don't use it (disabled optimization).
361 if (!Weights.empty()) {
362 weight_iterator WI = getWeightIterator(I);
363 Weights.erase(WI);
364 }
365
350366 Successors.erase(I);
351367 }
352368
353369 MachineBasicBlock::succ_iterator
354370 MachineBasicBlock::removeSuccessor(succ_iterator I) {
355371 assert(I != Successors.end() && "Not a current successor!");
372
373 // If Weight list is empty it means we don't use it (disabled optimization).
374 if (!Weights.empty()) {
375 weight_iterator WI = getWeightIterator(I);
376 Weights.erase(WI);
377 }
378
356379 (*I)->removePredecessor(this);
357380 return Successors.erase(I);
381 }
382
383 void MachineBasicBlock::replaceSuccessor(MachineBasicBlock *Old,
384 MachineBasicBlock *New) {
385 uint32_t weight = 0;
386 succ_iterator SI = std::find(Successors.begin(), Successors.end(), Old);
387
388 // If Weight list is empty it means we don't use it (disabled optimization).
389 if (!Weights.empty()) {
390 weight_iterator WI = getWeightIterator(SI);
391 weight = *WI;
392 }
393
394 // Update the successor information.
395 removeSuccessor(SI);
396 addSuccessor(New, weight);
358397 }
359398
360399 void MachineBasicBlock::addPredecessor(MachineBasicBlock *pred) {
373412
374413 while (!fromMBB->succ_empty()) {
375414 MachineBasicBlock *Succ = *fromMBB->succ_begin();
376 addSuccessor(Succ);
415 uint32_t weight = 0;
416
417
418 // If Weight list is empty it means we don't use it (disabled optimization).
419 if (!fromMBB->Weights.empty())
420 weight = *fromMBB->Weights.begin();
421
422 addSuccessor(Succ, weight);
377423 fromMBB->removeSuccessor(Succ);
378424 }
379425 }
636682 }
637683
638684 // Update the successor information.
639 removeSuccessor(Old);
640 addSuccessor(New);
685 replaceSuccessor(Old, New);
641686 }
642687
643688 /// CorrectExtraCFGEdges - Various pieces of code can cause excess edges in the
719764 return DL;
720765 }
721766
767 /// getSuccWeight - Return weight of the edge from this block to MBB.
768 ///
769 uint32_t MachineBasicBlock::getSuccWeight(MachineBasicBlock *succ) {
770 succ_iterator I = std::find(Successors.begin(), Successors.end(), succ);
771 return *getWeightIterator(I);
772 }
773
774 /// getWeightIterator - Return wight iterator corresonding to the I successor
775 /// iterator
776 MachineBasicBlock::weight_iterator MachineBasicBlock::
777 getWeightIterator(MachineBasicBlock::succ_iterator I) {
778 assert(Weights.size() == Successors.size() && "Async weight list!");
779 size_t index = std::distance(Successors.begin(), I);
780 assert(index < Weights.size() && "Not a current successor!");
781 return Weights.begin() + index;
782 }
783
722784 void llvm::WriteAsOperand(raw_ostream &OS, const MachineBasicBlock *MBB,
723785 bool t) {
724786 OS << "BB#" << MBB->getNumber();
0 //===- MachineBranchProbabilityInfo.cpp - Machine Branch Probability Info -===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This analysis uses probability info stored in Machine Basic Blocks.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Instructions.h"
14 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
15 #include "llvm/CodeGen/MachineBasicBlock.h"
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/raw_ostream.h"
18
19 using namespace llvm;
20
21 INITIALIZE_PASS_BEGIN(MachineBranchProbabilityInfo, "machine-branch-prob",
22 "Machine Branch Probability Analysis", false, true)
23 INITIALIZE_PASS_END(MachineBranchProbabilityInfo, "machine-branch-prob",
24 "Machine Branch Probability Analysis", false, true)
25
26 char MachineBranchProbabilityInfo::ID = 0;
27
28 uint32_t MachineBranchProbabilityInfo::
29 getSumForBlock(MachineBasicBlock *MBB) const {
30 uint32_t Sum = 0;
31
32 for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
33 E = MBB->succ_end(); I != E; ++I) {
34 MachineBasicBlock *Succ = *I;
35 uint32_t Weight = getEdgeWeight(MBB, Succ);
36 uint32_t PrevSum = Sum;
37
38 Sum += Weight;
39 assert(Sum > PrevSum); (void) PrevSum;
40 }
41
42 return Sum;
43 }
44
45 uint32_t
46 MachineBranchProbabilityInfo::getEdgeWeight(MachineBasicBlock *Src,
47 MachineBasicBlock *Dst) const {
48 uint32_t Weight = Src->getSuccWeight(Dst);
49 if (!Weight)
50 return DEFAULT_WEIGHT;
51 return Weight;
52 }
53
54 bool MachineBranchProbabilityInfo::isEdgeHot(MachineBasicBlock *Src,
55 MachineBasicBlock *Dst) const {
56 // Hot probability is at least 4/5 = 80%
57 uint32_t Weight = getEdgeWeight(Src, Dst);
58 uint32_t Sum = getSumForBlock(Src);
59
60 // FIXME: Implement BranchProbability::compare then change this code to
61 // compare this BranchProbability against a static "hot" BranchProbability.
62 return (uint64_t)Weight * 5 > (uint64_t)Sum * 4;
63 }
64
65 MachineBasicBlock *
66 MachineBranchProbabilityInfo::getHotSucc(MachineBasicBlock *MBB) const {
67 uint32_t Sum = 0;
68 uint32_t MaxWeight = 0;
69 MachineBasicBlock *MaxSucc = 0;
70
71 for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
72 E = MBB->succ_end(); I != E; ++I) {
73 MachineBasicBlock *Succ = *I;
74 uint32_t Weight = getEdgeWeight(MBB, Succ);
75 uint32_t PrevSum = Sum;
76
77 Sum += Weight;
78 assert(Sum > PrevSum); (void) PrevSum;
79
80 if (Weight > MaxWeight) {
81 MaxWeight = Weight;
82 MaxSucc = Succ;
83 }
84 }
85
86 // FIXME: Use BranchProbability::compare.
87 if ((uint64_t)MaxWeight * 5 >= (uint64_t)Sum * 4)
88 return MaxSucc;
89
90 return 0;
91 }
92
93 BranchProbability
94 MachineBranchProbabilityInfo::getEdgeProbability(MachineBasicBlock *Src,
95 MachineBasicBlock *Dst) const {
96 uint32_t N = getEdgeWeight(Src, Dst);
97 uint32_t D = getSumForBlock(Src);
98
99 return BranchProbability(N, D);
100 }
101
102 raw_ostream &MachineBranchProbabilityInfo::
103 printEdgeProbability(raw_ostream &OS, MachineBasicBlock *Src,
104 MachineBasicBlock *Dst) const {
105
106 const BranchProbability Prob = getEdgeProbability(Src, Dst);
107 OS << "edge MBB#" << Src->getNumber() << " -> MBB#" << Dst->getNumber()
108 << " probability is " << Prob
109 << (isEdgeHot(Src, Dst) ? " [HOT edge]\n" : "\n");
110
111 return OS;
112 }
12781278 return true;
12791279 }
12801280
1281 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1282 uint32_t SelectionDAGBuilder::getEdgeWeight(MachineBasicBlock *Src,
1283 MachineBasicBlock *Dst) {
1284 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1285 if (!BPI)
1286 return 0;
1287 BasicBlock *SrcBB = const_cast(Src->getBasicBlock());
1288 BasicBlock *DstBB = const_cast(Dst->getBasicBlock());
1289 return BPI->getEdgeWeight(SrcBB, DstBB);
1290 }
1291
1292 void SelectionDAGBuilder::addSuccessorWithWeight(MachineBasicBlock *Src,
1293 MachineBasicBlock *Dst) {
1294 uint32_t weight = getEdgeWeight(Src, Dst);
1295 Src->addSuccessor(Dst, weight);
1296 }
1297
1298
12811299 static bool InBlock(const Value *V, const BasicBlock *BB) {
12821300 if (const Instruction *I = dyn_cast(V))
12831301 return I->getParent() == BB;
15471565 }
15481566
15491567 // Update successor info
1550 SwitchBB->addSuccessor(CB.TrueBB);
1551 SwitchBB->addSuccessor(CB.FalseBB);
1568 addSuccessorWithWeight(SwitchBB, CB.TrueBB);
1569 addSuccessorWithWeight(SwitchBB, CB.FalseBB);
15521570
15531571 // Set NextBlock to be the MBB immediately after the current one, if any.
15541572 // This is used to avoid emitting unnecessary branches to the next block.
16921710
16931711 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
16941712
1695 SwitchBB->addSuccessor(B.Default);
1696 SwitchBB->addSuccessor(MBB);
1713 addSuccessorWithWeight(SwitchBB, B.Default);
1714 addSuccessorWithWeight(SwitchBB, MBB);
16971715
16981716 SDValue BrRange = DAG.getNode(ISD::BRCOND, getCurDebugLoc(),
16991717 MVT::Other, CopyTo, RangeCmp,
17381756 ISD::SETNE);
17391757 }
17401758
1741 SwitchBB->addSuccessor(B.TargetBB);
1742 SwitchBB->addSuccessor(NextMBB);
1759 addSuccessorWithWeight(SwitchBB, B.TargetBB);
1760 addSuccessorWithWeight(SwitchBB, NextMBB);
17431761
17441762 SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurDebugLoc(),
17451763 MVT::Other, getControlRoot(),
19791997 // table.
19801998 MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB);
19811999 CurMF->insert(BBI, JumpTableBB);
1982 CR.CaseBB->addSuccessor(Default);
1983 CR.CaseBB->addSuccessor(JumpTableBB);
2000
2001 addSuccessorWithWeight(CR.CaseBB, Default);
2002 addSuccessorWithWeight(CR.CaseBB, JumpTableBB);
19842003
19852004 // Build a vector of destination BBs, corresponding to each target
19862005 // of the jump table. If the value of the jump table slot corresponds to
20072026 E = DestBBs.end(); I != E; ++I) {
20082027 if (!SuccsHandled[(*I)->getNumber()]) {
20092028 SuccsHandled[(*I)->getNumber()] = true;
2010 JumpTableBB->addSuccessor(*I);
2029 addSuccessorWithWeight(JumpTableBB, *I);
20112030 }
20122031 }
20132032
24262445 succs.push_back(I.getSuccessor(i));
24272446 array_pod_sort(succs.begin(), succs.end());
24282447 succs.erase(std::unique(succs.begin(), succs.end()), succs.end());
2429 for (unsigned i = 0, e = succs.size(); i != e; ++i)
2430 IndirectBrMBB->addSuccessor(FuncInfo.MBBMap[succs[i]]);
2448 for (unsigned i = 0, e = succs.size(); i != e; ++i) {
2449 MachineBasicBlock *Succ = FuncInfo.MBBMap[succs[i]];
2450 addSuccessorWithWeight(IndirectBrMBB, Succ);
2451 }
24312452
24322453 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurDebugLoc(),
24332454 MVT::Other, getControlRoot(),
433433 const Value* SV,
434434 MachineBasicBlock* Default,
435435 MachineBasicBlock *SwitchBB);
436
437 uint32_t getEdgeWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst);
438 void addSuccessorWithWeight(MachineBasicBlock *Src, MachineBasicBlock *Dst);
436439 public:
437440 void visitSwitchCase(CaseBlock &CB,
438441 MachineBasicBlock *SwitchBB);
1616 #include "llvm/CodeGen/FunctionLoweringInfo.h"
1717 #include "llvm/CodeGen/SelectionDAGISel.h"
1818 #include "llvm/Analysis/AliasAnalysis.h"
19 #include "llvm/Analysis/BranchProbabilityInfo.h"
1920 #include "llvm/Analysis/DebugInfo.h"
2021 #include "llvm/Constants.h"
2122 #include "llvm/Function.h"
6768 EnableFastISelAbort("fast-isel-abort", cl::Hidden,
6869 cl::desc("Enable abort calls when \"fast\" instruction fails"));
6970
71 static cl::opt
72 UseMBPI("use-mbpi",
73 cl::desc("use Machine Branch Probability Info"),
74 cl::init(true), cl::Hidden);
75
7076 #ifndef NDEBUG
7177 static cl::opt
7278 ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
185191 DAGSize(0) {
186192 initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());
187193 initializeAliasAnalysisAnalysisGroup(*PassRegistry::getPassRegistry());
194 initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry());
188195 }
189196
190197 SelectionDAGISel::~SelectionDAGISel() {
198205 AU.addPreserved();
199206 AU.addRequired();
200207 AU.addPreserved();
208 if (UseMBPI && OptLevel != CodeGenOpt::None)
209 AU.addRequired();
201210 MachineFunctionPass::getAnalysisUsage(AU);
202211 }
203212
261270
262271 CurDAG->init(*MF);
263272 FuncInfo->set(Fn, *MF);
273
274 if (UseMBPI && OptLevel != CodeGenOpt::None)
275 FuncInfo->BPI = &getAnalysis();
276 else
277 FuncInfo->BPI = 0;
278
264279 SDB->init(GFI, *AA);
265280
266281 SelectAllBasicBlocks(Fn);