llvm.org GIT mirror llvm / de1c9bb
Remove return heuristics from the static branch probabilities, and introduce no-return or unreachable heuristics. The return heuristics from the Ball and Larus paper don't work well in practice as they pessimize early return paths. The only good hitrate return heuristics are those for: - NULL return - Constant return - negative integer return Only the last of these three can possibly require significant code for the returning block, and even the last is fairly rare and usually also a constant. As a consequence, even for the cold return paths, there is little code on that return path, and so little code density to be gained by sinking it. The places where sinking these blocks is valuable (inner loops) will already be weighted appropriately as the edge is a loop-exit branch. All of this aside, early returns are nearly as common as all three of these return categories, and should actually be predicted as taken! Rather than muddy the waters of the static predictions, just remain silent on returns and let the CFG itself dictate any layout or other issues. However, the return heuristic was flagging one very important case: unreachable. Unfortunately it still gave a 1/4 chance of the branch-to-unreachable occuring. It also didn't do a rigorous job of finding those blocks which post-dominate an unreachable block. This patch builds a more powerful analysis that should flag all branches to blocks known to then reach unreachable. It also has better worst-case runtime complexity by not looping through successors for each block. The previous code would perform an N^2 walk in the event of a single entry block branching to N successors with a switch where each successor falls through to the next and they finally fall through to a return. Test case added for noreturn heuristics. Also doxygen comments improved along the way. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@142793 91177308-0d34-0410-b5e6-96231b3b80d8 Chandler Carruth 8 years ago
3 changed file(s) with 165 addition(s) and 79 deletion(s). Raw diff Collapse all Expand all
1616 #include "llvm/InitializePasses.h"
1717 #include "llvm/Pass.h"
1818 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/SmallPtrSet.h"
1920 #include "llvm/Support/BranchProbability.h"
2021
2122 namespace llvm {
108109 /// \brief Track the last function we run over for printing.
109110 Function *LastF;
110111
112 /// \brief Track the set of blocks directly succeeded by a returning block.
113 SmallPtrSet PostDominatedByUnreachable;
114
111115 /// \brief Get sum of the block successors' weights.
112116 uint32_t getSumForBlock(const BasicBlock *BB) const;
113117
118 bool calcUnreachableHeuristics(BasicBlock *BB);
114119 bool calcMetadataWeights(BasicBlock *BB);
115 bool calcReturnHeuristics(BasicBlock *BB);
116120 bool calcPointerHeuristics(BasicBlock *BB);
117121 bool calcLoopBranchHeuristics(BasicBlock *BB);
118122 bool calcZeroHeuristics(BasicBlock *BB);
1717 #include "llvm/Metadata.h"
1818 #include "llvm/Analysis/BranchProbabilityInfo.h"
1919 #include "llvm/Analysis/LoopInfo.h"
20 #include "llvm/ADT/PostOrderIterator.h"
2021 #include "llvm/Support/CFG.h"
2122 #include "llvm/Support/Debug.h"
2223
5354 static const uint32_t LBH_TAKEN_WEIGHT = 124;
5455 static const uint32_t LBH_NONTAKEN_WEIGHT = 4;
5556
56 static const uint32_t RH_TAKEN_WEIGHT = 24;
57 static const uint32_t RH_NONTAKEN_WEIGHT = 8;
57 /// \brief Unreachable-terminating branch taken weight.
58 ///
59 /// This is the weight for a branch being taken to a block that terminates
60 /// (eventually) in unreachable. These are predicted as unlikely as possible.
61 static const uint32_t UR_TAKEN_WEIGHT = 1;
62
63 /// \brief Unreachable-terminating branch not-taken weight.
64 ///
65 /// This is the weight for a branch not being taken toward a block that
66 /// terminates (eventually) in unreachable. Such a branch is essentially never
67 /// taken.
68 static const uint32_t UR_NONTAKEN_WEIGHT = 1023;
5869
5970 static const uint32_t PH_TAKEN_WEIGHT = 20;
6071 static const uint32_t PH_NONTAKEN_WEIGHT = 12;
7283 // Minimum weight of an edge. Please note, that weight is NEVER 0.
7384 static const uint32_t MIN_WEIGHT = 1;
7485
75 // Return TRUE if BB leads directly to a Return Instruction.
76 static bool isReturningBlock(BasicBlock *BB) {
77 SmallPtrSet Visited;
78
79 while (true) {
80 TerminatorInst *TI = BB->getTerminator();
81 if (isa(TI))
82 return true;
83
84 if (TI->getNumSuccessors() > 1)
85 break;
86
87 // It is unreachable block which we can consider as a return instruction.
88 if (TI->getNumSuccessors() == 0)
89 return true;
90
91 Visited.insert(BB);
92 BB = TI->getSuccessor(0);
93
94 // Stop if cycle is detected.
95 if (Visited.count(BB))
96 return false;
97 }
98
99 return false;
100 }
101
10286 static uint32_t getMaxWeightFor(BasicBlock *BB) {
10387 return UINT32_MAX / BB->getTerminator()->getNumSuccessors();
10488 }
10589
90
91 /// \brief Calculate edge weights for successors lead to unreachable.
92 ///
93 /// Predict that a successor which leads necessarily to an
94 /// unreachable-terminated block as extremely unlikely.
95 bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) {
96 TerminatorInst *TI = BB->getTerminator();
97 if (TI->getNumSuccessors() == 0) {
98 if (isa(TI))
99 PostDominatedByUnreachable.insert(BB);
100 return false;
101 }
102
103 SmallPtrSet UnreachableEdges;
104 SmallPtrSet ReachableEdges;
105
106 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
107 if (PostDominatedByUnreachable.count(*I))
108 UnreachableEdges.insert(*I);
109 else
110 ReachableEdges.insert(*I);
111 }
112
113 // If all successors are in the set of blocks post-dominated by unreachable,
114 // this block is too.
115 if (UnreachableEdges.size() == TI->getNumSuccessors())
116 PostDominatedByUnreachable.insert(BB);
117
118 // Skip probabilities if this block has a single successor or if all were
119 // reachable.
120 if (TI->getNumSuccessors() == 1 || UnreachableEdges.empty())
121 return false;
122
123 uint32_t UnreachableWeight =
124 std::max(UR_TAKEN_WEIGHT / UnreachableEdges.size(), MIN_WEIGHT);
125 for (SmallPtrSet::iterator I = UnreachableEdges.begin(),
126 E = UnreachableEdges.end();
127 I != E; ++I)
128 setEdgeWeight(BB, *I, UnreachableWeight);
129
130 if (ReachableEdges.empty())
131 return true;
132 uint32_t ReachableWeight =
133 std::max(UR_NONTAKEN_WEIGHT / ReachableEdges.size(), NORMAL_WEIGHT);
134 for (SmallPtrSet::iterator I = ReachableEdges.begin(),
135 E = ReachableEdges.end();
136 I != E; ++I)
137 setEdgeWeight(BB, *I, ReachableWeight);
138
139 return true;
140 }
106141
107142 // Propagate existing explicit probabilities from either profile data or
108143 // 'expect' intrinsic processing.
140175 setEdgeWeight(BB, TI->getSuccessor(i), Weights[i]);
141176
142177 return true;
143 }
144
145 // Calculate Edge Weights using "Return Heuristics". Predict a successor which
146 // leads directly to Return Instruction will not be taken.
147 bool BranchProbabilityInfo::calcReturnHeuristics(BasicBlock *BB){
148 if (BB->getTerminator()->getNumSuccessors() == 1)
149 return false;
150
151 SmallPtrSet ReturningEdges;
152 SmallPtrSet StayEdges;
153
154 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
155 BasicBlock *Succ = *I;
156 if (isReturningBlock(Succ))
157 ReturningEdges.insert(Succ);
158 else
159 StayEdges.insert(Succ);
160 }
161
162 if (uint32_t numStayEdges = StayEdges.size()) {
163 uint32_t stayWeight = RH_TAKEN_WEIGHT / numStayEdges;
164 if (stayWeight < NORMAL_WEIGHT)
165 stayWeight = NORMAL_WEIGHT;
166
167 for (SmallPtrSet::iterator I = StayEdges.begin(),
168 E = StayEdges.end(); I != E; ++I)
169 setEdgeWeight(BB, *I, stayWeight);
170 }
171
172 if (uint32_t numRetEdges = ReturningEdges.size()) {
173 uint32_t retWeight = RH_NONTAKEN_WEIGHT / numRetEdges;
174 if (retWeight < MIN_WEIGHT)
175 retWeight = MIN_WEIGHT;
176 for (SmallPtrSet::iterator I = ReturningEdges.begin(),
177 E = ReturningEdges.end(); I != E; ++I) {
178 setEdgeWeight(BB, *I, retWeight);
179 }
180 }
181
182 return ReturningEdges.size() > 0;
183178 }
184179
185180 // Calculate Edge Weights using "Pointer Heuristics". Predict a comparsion
389384 bool BranchProbabilityInfo::runOnFunction(Function &F) {
390385 LastF = &F; // Store the last function we ran on for printing.
391386 LI = &getAnalysis();
392
393 for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
394 if (calcMetadataWeights(I))
387 assert(PostDominatedByUnreachable.empty());
388
389 // Walk the basic blocks in post-order so that we can build up state about
390 // the successors of a block iteratively.
391 for (po_iterator I = po_begin(&F.getEntryBlock()),
392 E = po_end(&F.getEntryBlock());
393 I != E; ++I) {
394 DEBUG(dbgs() << "Computing probabilities for " << I->getName() << "\n");
395 if (calcUnreachableHeuristics(*I))
395396 continue;
396 if (calcLoopBranchHeuristics(I))
397 if (calcMetadataWeights(*I))
397398 continue;
398 if (calcReturnHeuristics(I))
399 if (calcLoopBranchHeuristics(*I))
399400 continue;
400 if (calcPointerHeuristics(I))
401 if (calcPointerHeuristics(*I))
401402 continue;
402 if (calcZeroHeuristics(I))
403 if (calcZeroHeuristics(*I))
403404 continue;
404 calcFloatingPointHeuristics(I);
405 }
405 calcFloatingPointHeuristics(*I);
406 }
407
408 PostDominatedByUnreachable.clear();
406409 return false;
407410 }
408411
0 ; Test the static branch probability heuristics for no-return functions.
1 ; RUN: opt < %s -analyze -branch-prob | FileCheck %s
2
3 declare void @abort() noreturn
4
5 define i32 @test1(i32 %a, i32 %b) {
6 ; CHECK: Printing analysis {{.*}} for function 'test1'
7 entry:
8 %cond = icmp eq i32 %a, 42
9 br i1 %cond, label %exit, label %abort
10 ; CHECK: edge entry -> exit probability is 1023 / 1024
11 ; CHECK: edge entry -> abort probability is 1 / 1024
12
13 abort:
14 call void @abort() noreturn
15 unreachable
16
17 exit:
18 ret i32 %b
19 }
20
21 define i32 @test2(i32 %a, i32 %b) {
22 ; CHECK: Printing analysis {{.*}} for function 'test2'
23 entry:
24 switch i32 %a, label %exit [i32 1, label %case_a
25 i32 2, label %case_b
26 i32 3, label %case_c
27 i32 4, label %case_d]
28 ; CHECK: edge entry -> exit probability is 1023 / 1027
29 ; CHECK: edge entry -> case_a probability is 1 / 1027
30 ; CHECK: edge entry -> case_b probability is 1 / 1027
31 ; CHECK: edge entry -> case_c probability is 1 / 1027
32 ; CHECK: edge entry -> case_d probability is 1 / 1027
33
34 case_a:
35 br label %case_b
36
37 case_b:
38 br label %case_c
39
40 case_c:
41 br label %case_d
42
43 case_d:
44 call void @abort() noreturn
45 unreachable
46
47 exit:
48 ret i32 %b
49 }
50
51 define i32 @test3(i32 %a, i32 %b) {
52 ; CHECK: Printing analysis {{.*}} for function 'test3'
53 ; Make sure we unify across multiple conditional branches.
54 entry:
55 %cond1 = icmp eq i32 %a, 42
56 br i1 %cond1, label %exit, label %dom
57 ; CHECK: edge entry -> exit probability is 1023 / 1024
58 ; CHECK: edge entry -> dom probability is 1 / 1024
59
60 dom:
61 %cond2 = icmp ult i32 %a, 42
62 br i1 %cond2, label %idom1, label %idom2
63 ; CHECK: edge dom -> idom1 probability is 1 / 2
64 ; CHECK: edge dom -> idom2 probability is 1 / 2
65
66 idom1:
67 br label %abort
68
69 idom2:
70 br label %abort
71
72 abort:
73 call void @abort() noreturn
74 unreachable
75
76 exit:
77 ret i32 %b
78 }