llvm.org GIT mirror llvm / f289df2
Branch profiling: floating-point avoidance. Patch by: Jakub Staszak! Introduces BranchProbability. Changes unsigned to uint32_t all over and uint64_t only when overflow is expected. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@132867 91177308-0d34-0410-b5e6-96231b3b80d8 Andrew Trick 9 years ago
5 changed file(s) with 179 addition(s) and 65 deletion(s). Raw diff Collapse all Expand all
1414 #define LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H
1515
1616 #include "llvm/InitializePasses.h"
17 #include "llvm/Support/BranchProbability.h"
1718 #include "llvm/Analysis/LoopInfo.h"
18 #include "llvm/Support/Debug.h"
1919
2020 namespace llvm {
21
22 class raw_ostream;
2123
2224 class BranchProbabilityInfo : public FunctionPass {
2325
2426 // Default weight value. Used when we don't have information about the edge.
25 static const unsigned int DEFAULT_WEIGHT = 16;
27 static const uint32_t DEFAULT_WEIGHT = 16;
2628
2729 typedef std::pair Edge;
2830
29 DenseMapnsigned> Weights;
31 DenseMapint32_t> Weights;
32
33 // Get sum of the block successors' weights.
34 uint32_t getSumForBlock(BasicBlock *BB) const;
3035
3136 public:
3237 static char ID;
4247
4348 bool runOnFunction(Function &F);
4449
45 // Returned value is between 1 and UINT_MAX. Look at BranchProbabilityInfo.cpp
46 // for details.
47 unsigned getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const;
50 // Returned value is between 1 and UINT32_MAX. Look at
51 // BranchProbabilityInfo.cpp for details.
52 uint32_t getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const;
4853
4954 // Look at BranchProbabilityInfo.cpp for details. Use it with caution!
50 void setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, unsigned Weight);
55 void setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, uint32_t Weight);
5156
5257 // A 'Hot' edge is an edge which probability is >= 80%.
5358 bool isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const;
5560 // Return a hot successor for the block BB or null if there isn't one.
5661 BasicBlock *getHotSucc(BasicBlock *BB) const;
5762
63 // Return a probability as a fraction between 0 (0% probability) and
64 // 1 (100% probability), however the value is never equal to 0, and can be 1
65 // only iff SRC block has only one successor.
66 BranchProbability getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const;
67
5868 // Print value between 0 (0% probability) and 1 (100% probability),
5969 // however the value is never equal to 0, and can be 1 only iff SRC block
6070 // has only one successor.
6171 raw_ostream &printEdgeProbability(raw_ostream &OS, BasicBlock *Src,
62 BasicBlock *Dst) const;
72 BasicBlock *Dst) const;
6373 };
6474
6575 }
0 //===- BranchProbability.h - Branch Probability Analysis --------*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Definition of BranchProbability shared by IR and Machine Instructions.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #ifndef LLVM_SUPPORT_BRANCHPROBABILITY_H
14 #define LLVM_SUPPORT_BRANCHPROBABILITY_H
15
16 #include "llvm/Support/DataTypes.h"
17
18 namespace llvm {
19
20 class raw_ostream;
21 class BranchProbabilityInfo;
22 class MachineBranchProbabilityInfo;
23 class MachineBasicBlock;
24
25 // This class represents Branch Probability as a non-negative fraction.
26 class BranchProbability {
27 friend class BranchProbabilityInfo;
28 friend class MachineBranchProbabilityInfo;
29 friend class MachineBasicBlock;
30
31 // Numerator
32 uint32_t N;
33
34 // Denominator
35 uint32_t D;
36
37 BranchProbability(uint32_t n, uint32_t d);
38
39 public:
40 raw_ostream &print(raw_ostream &OS) const;
41
42 void dump() const;
43 };
44
45 raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob);
46
47 }
48
49 #endif
1212
1313 #include "llvm/Instructions.h"
1414 #include "llvm/Analysis/BranchProbabilityInfo.h"
15 #include
15 #include "llvm/Support/Debug.h"
1616
1717 using namespace llvm;
1818
3333
3434 typedef std::pair Edge;
3535
36 DenseMapnsigned> *Weights;
36 DenseMapint32_t> *Weights;
3737
3838 BranchProbabilityInfo *BP;
3939
6161 // Probability of the edge BB2->BB1 = 128 / (128 + 4) = 0.9696..
6262 // Probability of the edge BB2->BB3 = 4 / (128 + 4) = 0.0303..
6363
64 static const unsigned int LBH_TAKEN_WEIGHT = 128;
65 static const unsigned int LBH_NONTAKEN_WEIGHT = 4;
64 static const uint32_t LBH_TAKEN_WEIGHT = 128;
65 static const uint32_t LBH_NONTAKEN_WEIGHT = 4;
6666
6767 // Standard weight value. Used when none of the heuristics set weight for
6868 // the edge.
69 static const unsigned int NORMAL_WEIGHT = 16;
69 static const uint32_t NORMAL_WEIGHT = 16;
7070
7171 // Minimum weight of an edge. Please note, that weight is NEVER 0.
72 static const unsigned int MIN_WEIGHT = 1;
72 static const uint32_t MIN_WEIGHT = 1;
7373
7474 // Return TRUE if BB leads directly to a Return Instruction.
7575 static bool isReturningBlock(BasicBlock *BB) {
100100
101101 // Multiply Edge Weight by two.
102102 void incEdgeWeight(BasicBlock *Src, BasicBlock *Dst) {
103 unsigned Weight = BP->getEdgeWeight(Src, Dst);
104 unsigned MaxWeight = getMaxWeightFor(Src);
103 uint32_t Weight = BP->getEdgeWeight(Src, Dst);
104 uint32_t MaxWeight = getMaxWeightFor(Src);
105105
106106 if (Weight * 2 > MaxWeight)
107107 BP->setEdgeWeight(Src, Dst, MaxWeight);
111111
112112 // Divide Edge Weight by two.
113113 void decEdgeWeight(BasicBlock *Src, BasicBlock *Dst) {
114 unsigned Weight = BP->getEdgeWeight(Src, Dst);
114 uint32_t Weight = BP->getEdgeWeight(Src, Dst);
115115
116116 assert(Weight > 0);
117117 if (Weight / 2 < MIN_WEIGHT)
121121 }
122122
123123
124 unsigned getMaxWeightFor(BasicBlock *BB) const {
125 return UINT_MAX / BB->getTerminator()->getNumSuccessors();
124 uint32_t getMaxWeightFor(BasicBlock *BB) const {
125 return UINT32_MAX / BB->getTerminator()->getNumSuccessors();
126126 }
127127
128128 public:
129 BranchProbabilityAnalysis(DenseMapnsigned> *W,
129 BranchProbabilityAnalysis(DenseMapint32_t> *W,
130130 BranchProbabilityInfo *BP, LoopInfo *LI)
131131 : Weights(W), BP(BP), LI(LI) {
132132 }
194194 // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges
195195 // as taken, exiting edges as not-taken.
196196 void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) {
197 unsigned numSuccs = BB->getTerminator()->getNumSuccessors();
197 uint32_t numSuccs = BB->getTerminator()->getNumSuccessors();
198198
199199 Loop *L = LI->getLoopFor(BB);
200200 if (!L)
212212 BackEdges.push_back(Succ);
213213 }
214214
215 if (unsigned numBackEdges = BackEdges.size()) {
216 unsigned backWeight = LBH_TAKEN_WEIGHT / numBackEdges;
215 if (uint32_t numBackEdges = BackEdges.size()) {
216 uint32_t backWeight = LBH_TAKEN_WEIGHT / numBackEdges;
217217 if (backWeight < NORMAL_WEIGHT)
218218 backWeight = NORMAL_WEIGHT;
219219
224224 }
225225 }
226226
227 unsigned numExitingEdges = ExitingEdges.size();
228 if (unsigned numNonExitingEdges = numSuccs - numExitingEdges) {
229 unsigned exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges;
227 uint32_t numExitingEdges = ExitingEdges.size();
228 if (uint32_t numNonExitingEdges = numSuccs - numExitingEdges) {
229 uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges;
230230 if (exitWeight < MIN_WEIGHT)
231231 exitWeight = MIN_WEIGHT;
232232
259259 bool BranchProbabilityInfo::runOnFunction(Function &F) {
260260 LoopInfo &LI = getAnalysis();
261261 BranchProbabilityAnalysis BPA(&Weights, this, &LI);
262 bool ret = BPA.runOnFunction(F);
263 return ret;
264 }
265
266 // TODO: This currently hardcodes 80% as a fraction 4/5. We will soon add a
267 // BranchProbability class to encapsulate the fractional probability and
268 // define a few static instances of the class for use as predefined thresholds.
269 bool BranchProbabilityInfo::isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const {
270 unsigned Sum = 0;
271 for (succ_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) {
262 return BPA.runOnFunction(F);
263 }
264
265 uint32_t BranchProbabilityInfo::getSumForBlock(BasicBlock *BB) const {
266 uint32_t Sum = 0;
267
268 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
272269 BasicBlock *Succ = *I;
273 unsigned Weight = getEdgeWeight(Src, Succ);
274 unsigned PrevSum = Sum;
270 uint32_t Weight = getEdgeWeight(BB, Succ);
271 uint32_t PrevSum = Sum;
275272
276273 Sum += Weight;
277274 assert(Sum > PrevSum); (void) PrevSum;
278275 }
279276
280 return getEdgeWeight(Src, Dst) * 5 > Sum * 4;
277 return Sum;
278 }
279
280 bool BranchProbabilityInfo::isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const {
281 // Hot probability is at least 4/5 = 80%
282 uint32_t Weight = getEdgeWeight(Src, Dst);
283 uint32_t Sum = getSumForBlock(Src);
284
285 // FIXME: Implement BranchProbability::compare then change this code to
286 // compare this BranchProbability against a static "hot" BranchProbability.
287 return (uint64_t)Weight * 5 > (uint64_t)Sum * 4;
281288 }
282289
283290 BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const {
284 unsigned Sum = 0;
285 unsigned MaxWeight = 0;
291 uint32_t Sum = 0;
292 uint32_t MaxWeight = 0;
286293 BasicBlock *MaxSucc = 0;
287294
288295 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
289296 BasicBlock *Succ = *I;
290 unsigned Weight = getEdgeWeight(BB, Succ);
291 unsigned PrevSum = Sum;
297 uint32_t Weight = getEdgeWeight(BB, Succ);
298 uint32_t PrevSum = Sum;
292299
293300 Sum += Weight;
294301 assert(Sum > PrevSum); (void) PrevSum;
299306 }
300307 }
301308
302 if (MaxWeight * 5 > Sum * 4)
309 // FIXME: Use BranchProbability::compare.
310 if ((uint64_t)MaxWeight * 5 > (uint64_t)Sum * 4)
303311 return MaxSucc;
304312
305313 return 0;
306314 }
307315
308316 // Return edge's weight. If can't find it, return DEFAULT_WEIGHT value.
309 unsigned
317 uint32_t
310318 BranchProbabilityInfo::getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const {
311319 Edge E(Src, Dst);
312 DenseMapnsigned>::const_iterator I = Weights.find(E);
320 DenseMapint32_t>::const_iterator I = Weights.find(E);
313321
314322 if (I != Weights.end())
315323 return I->second;
318326 }
319327
320328 void BranchProbabilityInfo::setEdgeWeight(BasicBlock *Src, BasicBlock *Dst,
321 unsigned Weight) {
329 uint32_t Weight) {
322330 Weights[std::make_pair(Src, Dst)] = Weight;
323 DEBUG(dbgs() << "setEdgeWeight: " << Src->getNameStr() << " -> "
324 << Dst->getNameStr() << " to " << Weight
325 << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n"));
331 DEBUG(dbgs() << "set edge " << Src->getNameStr() << " -> "
332 << Dst->getNameStr() << " weight to " << Weight
333 << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n"));
334 }
335
336
337 BranchProbability BranchProbabilityInfo::
338 getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const {
339
340 uint32_t N = getEdgeWeight(Src, Dst);
341 uint32_t D = getSumForBlock(Src);
342
343 return BranchProbability(N, D);
326344 }
327345
328346 raw_ostream &
329347 BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS, BasicBlock *Src,
330 BasicBlock *Dst) const {
331
332 unsigned Sum = 0;
333 for (succ_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) {
334 BasicBlock *Succ = *I;
335 unsigned Weight = getEdgeWeight(Src, Succ);
336 unsigned PrevSum = Sum;
337
338 Sum += Weight;
339 assert(Sum > PrevSum); (void) PrevSum;
340 }
341
342 double Prob = (double)getEdgeWeight(Src, Dst) / Sum;
343 OS << "probability (" << Src->getNameStr() << " --> " << Dst->getNameStr()
344 << ") = " << Prob << "\n";
348 BasicBlock *Dst) const {
349 BranchProbability Prob = getEdgeProbability(Src, Dst);
350
351 OS << "edge " << Src->getNameStr() << " -> " << Dst->getNameStr()
352 << " probability is " << Prob
353 << (isEdgeHot(Src, Dst) ? " [HOT edge]\n" : "\n");
345354
346355 return OS;
347356 }
0 //===-------------- lib/Support/BranchProbability.cpp -----------*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements Branch Probability class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Support/BranchProbability.h"
14 #include "llvm/Support/Debug.h"
15 #include "llvm/Support/raw_ostream.h"
16
17 using namespace llvm;
18
19 BranchProbability::BranchProbability(uint32_t n, uint32_t d) {
20 assert(d > 0 && "Denomiator cannot be 0!");
21 assert(n <= d && "Probability cannot be bigger than 1!");
22 N = n;
23 D = d;
24 }
25
26 raw_ostream &BranchProbability::print(raw_ostream &OS) const {
27 OS << N << " / " << D << " = " << ((double)N / D);
28 return OS;
29 }
30
31 void BranchProbability::dump() const {
32 print(dbgs());
33 dbgs() << "\n";
34 }
35
36 namespace llvm {
37
38 raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob) {
39 Prob.print(OS);
40 return OS;
41 }
42
43 }
88 APInt.cpp
99 APSInt.cpp
1010 Allocator.cpp
11 BranchProbability.cpp
1112 circular_raw_ostream.cpp
1213 CommandLine.cpp
1314 ConstantRange.cpp