llvm.org GIT mirror llvm / 1a710fd
BranchProb: modify the definition of an edge in BranchProbabilityInfo to handle the case of multiple edges from one block to another. A simple example is a switch statement with multiple values to the same destination. The definition of an edge is modified from a pair of blocks to a pair of PredBlock and an index into the successors. Also set the weight correctly when building SelectionDAG from LLVM IR, especially when converting a Switch. IntegersSubsetMapping is updated to calculate the weight for each cluster. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@162572 91177308-0d34-0410-b5e6-96231b3b80d8 Manman Ren 7 years ago
8 changed file(s) with 250 addition(s) and 113 deletion(s). Raw diff Collapse all Expand all
2727 ///
2828 /// This is a function analysis pass which provides information on the relative
2929 /// probabilities of each "edge" in the function's CFG where such an edge is
30 /// defined by a pair of basic blocks. The probability for a given block and
31 /// a successor block are always relative to the probabilities of the other
32 /// successor blocks. Another way of looking at it is that the probabilities
33 /// for a given block B and each of its successors should sum to exactly
34 /// one (100%).
30 /// defined by a pair (PredBlock and an index in the successors). The
31 /// probability of an edge from one block is always relative to the
32 /// probabilities of other edges from the block. The probabilites of all edges
33 /// from a block sum to exactly one (100%).
34 /// We use a pair (PredBlock and an index in the successors) to uniquely
35 /// identify an edge, since we can have multiple edges from Src to Dst.
36 /// As an example, we can have a switch which jumps to Dst with value 0 and
37 /// value 10.
3538 class BranchProbabilityInfo : public FunctionPass {
3639 public:
3740 static char ID;
5053 /// (0%) and one (100%) of this edge executing, relative to other edges
5154 /// leaving the 'Src' block. The returned probability is never zero, and can
5255 /// only be one if the source block has only one successor.
56 BranchProbability getEdgeProbability(const BasicBlock *Src,
57 unsigned IndexInSuccessors) const;
58
59 /// \brief Get the probability of going from Src to Dst.
60 ///
61 /// It returns the sum of all probabilities for edges from Src to Dst.
5362 BranchProbability getEdgeProbability(const BasicBlock *Src,
5463 const BasicBlock *Dst) const;
5564
7382 raw_ostream &printEdgeProbability(raw_ostream &OS, const BasicBlock *Src,
7483 const BasicBlock *Dst) const;
7584
76 /// \brief Get the raw edge weight calculated for the block pair.
85 /// \brief Get the raw edge weight calculated for the edge.
7786 ///
7887 /// This returns the raw edge weight. It is guaranteed to fall between 1 and
7988 /// UINT32_MAX. Note that the raw edge weight is not meaningful in isolation.
8089 /// This interface should be very carefully, and primarily by routines that
8190 /// are updating the analysis by later calling setEdgeWeight.
91 uint32_t getEdgeWeight(const BasicBlock *Src,
92 unsigned IndexInSuccessors) const;
93
94 /// \brief Get the raw edge weight calculated for the block pair.
95 ///
96 /// This returns the sum of all raw edge weights from Src to Dst.
97 /// It is guaranteed to fall between 1 and UINT32_MAX.
8298 uint32_t getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const;
8399
84 /// \brief Set the raw edge weight for the block pair.
100 /// \brief Set the raw edge weight for a given edge.
85101 ///
86 /// This allows a pass to explicitly set the edge weight for a block. It can
102 /// This allows a pass to explicitly set the edge weight for an edge. It can
87103 /// be used when updating the CFG to update and preserve the branch
88104 /// probability information. Read the implementation of how these edge
89105 /// weights are calculated carefully before using!
90 void setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst,
106 void setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors,
91107 uint32_t Weight);
92108
93109 private:
94 typedef std::pair Edge;
110 // Since we allow duplicate edges from one basic block to another, we use
111 // a pair (PredBlock and an index in the successors) to specify an edge.
112 typedef std::pair Edge;
95113
96114 // Default weight value. Used when we don't have information about the edge.
97115 // TODO: DEFAULT_WEIGHT makes sense during static predication, when none of
4141 struct RangeEx : public RangeTy {
4242 RangeEx() : Weight(1) {}
4343 RangeEx(const RangeTy &R) : RangeTy(R), Weight(1) {}
44 RangeEx(const RangeTy &R, unsigned W) : RangeTy(R), Weight(W) {}
4445 RangeEx(const IntTy &C) : RangeTy(C), Weight(1) {}
4546 RangeEx(const IntTy &L, const IntTy &H) : RangeTy(L, H), Weight(1) {}
4647 RangeEx(const IntTy &L, const IntTy &H, unsigned W) :
315316 Items.clear();
316317 const IntTy *Low = &OldItems.begin()->first.getLow();
317318 const IntTy *High = &OldItems.begin()->first.getHigh();
318 unsigned Weight = 1;
319 unsigned Weight = OldItems.begin()->first.Weight;
319320 SuccessorClass *Successor = OldItems.begin()->second;
320321 for (CaseItemIt j = OldItems.begin(), i = j++, e = OldItems.end();
321322 j != e; i = j++) {
322323 if (isJoinable(i, j)) {
323324 const IntTy *CurHigh = &j->first.getHigh();
324 ++Weight;
325 Weight += j->first.Weight;
325326 if (*CurHigh > *High)
326327 High = CurHigh;
327328 } else {
329330 add(R, Successor);
330331 Low = &j->first.getLow();
331332 High = &j->first.getHigh();
332 Weight = 1;
333 Weight = j->first.Weight;
333334 Successor = j->second;
334335 }
335336 }
361362
362363 /// Adds all ranges and values from given ranges set to the current
363364 /// mapping.
364 void add(const IntegersSubsetTy &CRS, SuccessorClass *S = 0) {
365 void add(const IntegersSubsetTy &CRS, SuccessorClass *S = 0,
366 unsigned Weight = 0) {
367 unsigned ItemWeight = 1;
368 if (Weight)
369 // Weight is associated with CRS, for now we perform a division to
370 // get the weight for each item.
371 ItemWeight = Weight / CRS.getNumItems();
365372 for (unsigned i = 0, e = CRS.getNumItems(); i < e; ++i) {
366373 RangeTy R = CRS.getItem(i);
367 add(R, S);
374 RangeEx REx(R, ItemWeight);
375 add(REx, S);
368376 }
369377 }
370378
114114 return false;
115115 }
116116
117 SmallPtrSet UnreachableEdges;
118 SmallPtrSet ReachableEdges;
117 SmallVector UnreachableEdges;
118 SmallVector ReachableEdges;
119119
120120 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
121121 if (PostDominatedByUnreachable.count(*I))
122 UnreachableEdges.insert(*I);
122 UnreachableEdges.push_back(I.getSuccessorIndex());
123123 else
124 ReachableEdges.insert(*I);
124 ReachableEdges.push_back(I.getSuccessorIndex());
125125 }
126126
127127 // If all successors are in the set of blocks post-dominated by unreachable,
135135 return false;
136136
137137 uint32_t UnreachableWeight =
138 std::max(UR_TAKEN_WEIGHT / UnreachableEdges.size(), MIN_WEIGHT);
139 for (SmallPtrSet::iterator I = UnreachableEdges.begin(),
140 E = UnreachableEdges.end();
138 std::max(UR_TAKEN_WEIGHT / (unsigned)UnreachableEdges.size(), MIN_WEIGHT);
139 for (SmallVector::iterator I = UnreachableEdges.begin(),
140 E = UnreachableEdges.end();
141141 I != E; ++I)
142142 setEdgeWeight(BB, *I, UnreachableWeight);
143143
144144 if (ReachableEdges.empty())
145145 return true;
146146 uint32_t ReachableWeight =
147 std::max(UR_NONTAKEN_WEIGHT / ReachableEdges.size(), NORMAL_WEIGHT);
148 for (SmallPtrSet::iterator I = ReachableEdges.begin(),
149 E = ReachableEdges.end();
147 std::max(UR_NONTAKEN_WEIGHT / (unsigned)ReachableEdges.size(),
148 NORMAL_WEIGHT);
149 for (SmallVector::iterator I = ReachableEdges.begin(),
150 E = ReachableEdges.end();
150151 I != E; ++I)
151152 setEdgeWeight(BB, *I, ReachableWeight);
152153
186187 }
187188 assert(Weights.size() == TI->getNumSuccessors() && "Checked above");
188189 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
189 setEdgeWeight(BB, TI->getSuccessor(i), Weights[i]);
190 setEdgeWeight(BB, i, Weights[i]);
190191
191192 return true;
192193 }
210211
211212 assert(CI->getOperand(1)->getType()->isPointerTy());
212213
213 BasicBlock *Taken = BI->getSuccessor(0);
214 BasicBlock *NonTaken = BI->getSuccessor(1);
215
216214 // p != 0 -> isProb = true
217215 // p == 0 -> isProb = false
218216 // p != q -> isProb = true
219217 // p == q -> isProb = false;
218 unsigned TakenIdx = 0, NonTakenIdx = 1;
220219 bool isProb = CI->getPredicate() == ICmpInst::ICMP_NE;
221220 if (!isProb)
222 std::swap(Taken, NonTaken);
223
224 setEdgeWeight(BB, Taken, PH_TAKEN_WEIGHT);
225 setEdgeWeight(BB, NonTaken, PH_NONTAKEN_WEIGHT);
221 std::swap(TakenIdx, NonTakenIdx);
222
223 setEdgeWeight(BB, TakenIdx, PH_TAKEN_WEIGHT);
224 setEdgeWeight(BB, NonTakenIdx, PH_NONTAKEN_WEIGHT);
226225 return true;
227226 }
228227
233232 if (!L)
234233 return false;
235234
236 SmallPtrSet BackEdges;
237 SmallPtrSet ExitingEdges;
238 SmallPtrSet InEdges; // Edges from header to the loop.
235 SmallVector BackEdges;
236 SmallVector ExitingEdges;
237 SmallVector InEdges; // Edges from header to the loop.
239238
240239 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
241240 if (!L->contains(*I))
242 ExitingEdges.insert(*I);
241 ExitingEdges.push_back(I.getSuccessorIndex());
243242 else if (L->getHeader() == *I)
244 BackEdges.insert(*I);
243 BackEdges.push_back(I.getSuccessorIndex());
245244 else
246 InEdges.insert(*I);
245 InEdges.push_back(I.getSuccessorIndex());
247246 }
248247
249248 if (uint32_t numBackEdges = BackEdges.size()) {
251250 if (backWeight < NORMAL_WEIGHT)
252251 backWeight = NORMAL_WEIGHT;
253252
254 for (SmallPtrSet, 8>::iterator EI = BackEdges.begin(),
253 for (SmallVector, 8>::iterator EI = BackEdges.begin(),
255254 EE = BackEdges.end(); EI != EE; ++EI) {
256 BasicBlock *Back = *EI;
257 setEdgeWeight(BB, Back, backWeight);
255 setEdgeWeight(BB, *EI, backWeight);
258256 }
259257 }
260258
263261 if (inWeight < NORMAL_WEIGHT)
264262 inWeight = NORMAL_WEIGHT;
265263
266 for (SmallPtrSet, 8>::iterator EI = InEdges.begin(),
264 for (SmallVector, 8>::iterator EI = InEdges.begin(),
267265 EE = InEdges.end(); EI != EE; ++EI) {
268 BasicBlock *Back = *EI;
269 setEdgeWeight(BB, Back, inWeight);
266 setEdgeWeight(BB, *EI, inWeight);
270267 }
271268 }
272269
275272 if (exitWeight < MIN_WEIGHT)
276273 exitWeight = MIN_WEIGHT;
277274
278 for (SmallPtrSet, 8>::iterator EI = ExitingEdges.begin(),
275 for (SmallVector, 8>::iterator EI = ExitingEdges.begin(),
279276 EE = ExitingEdges.end(); EI != EE; ++EI) {
280 BasicBlock *Exiting = *EI;
281 setEdgeWeight(BB, Exiting, exitWeight);
277 setEdgeWeight(BB, *EI, exitWeight);
282278 }
283279 }
284280
334330 return false;
335331 }
336332
337 BasicBlock *Taken = BI->getSuccessor(0);
338 BasicBlock *NonTaken = BI->getSuccessor(1);
333 unsigned TakenIdx = 0, NonTakenIdx = 1;
339334
340335 if (!isProb)
341 std::swap(Taken, NonTaken);
342
343 setEdgeWeight(BB, Taken, ZH_TAKEN_WEIGHT);
344 setEdgeWeight(BB, NonTaken, ZH_NONTAKEN_WEIGHT);
336 std::swap(TakenIdx, NonTakenIdx);
337
338 setEdgeWeight(BB, TakenIdx, ZH_TAKEN_WEIGHT);
339 setEdgeWeight(BB, NonTakenIdx, ZH_NONTAKEN_WEIGHT);
345340
346341 return true;
347342 }
371366 return false;
372367 }
373368
374 BasicBlock *Taken = BI->getSuccessor(0);
375 BasicBlock *NonTaken = BI->getSuccessor(1);
369 unsigned TakenIdx = 0, NonTakenIdx = 1;
376370
377371 if (!isProb)
378 std::swap(Taken, NonTaken);
379
380 setEdgeWeight(BB, Taken, FPH_TAKEN_WEIGHT);
381 setEdgeWeight(BB, NonTaken, FPH_NONTAKEN_WEIGHT);
372 std::swap(TakenIdx, NonTakenIdx);
373
374 setEdgeWeight(BB, TakenIdx, FPH_TAKEN_WEIGHT);
375 setEdgeWeight(BB, NonTakenIdx, FPH_NONTAKEN_WEIGHT);
382376
383377 return true;
384378 }
388382 if (!II)
389383 return false;
390384
391 BasicBlock *Normal = II->getNormalDest();
392 BasicBlock *Unwind = II->getUnwindDest();
393
394 setEdgeWeight(BB, Normal, IH_TAKEN_WEIGHT);
395 setEdgeWeight(BB, Unwind, IH_NONTAKEN_WEIGHT);
385 setEdgeWeight(BB, 0/*Index for Normal*/, IH_TAKEN_WEIGHT);
386 setEdgeWeight(BB, 1/*Index for Unwind*/, IH_NONTAKEN_WEIGHT);
396387 return true;
397388 }
398389
449440 uint32_t Sum = 0;
450441
451442 for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
452 const BasicBlock *Succ = *I;
453 uint32_t Weight = getEdgeWeight(BB, Succ);
443 uint32_t Weight = getEdgeWeight(BB, I.getSuccessorIndex());
454444 uint32_t PrevSum = Sum;
455445
456446 Sum += Weight;
493483 return 0;
494484 }
495485
496 // Return edge's weight. If can't find it, return DEFAULT_WEIGHT value.
486 /// Get the raw edge weight for the edge. If can't find it, return
487 /// DEFAULT_WEIGHT value. Here an edge is specified using PredBlock and an index
488 /// to the successors.
489 uint32_t BranchProbabilityInfo::
490 getEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors) const {
491 DenseMap::const_iterator I =
492 Weights.find(std::make_pair(Src, IndexInSuccessors));
493
494 if (I != Weights.end())
495 return I->second;
496
497 return DEFAULT_WEIGHT;
498 }
499
500 /// Get the raw edge weight calculated for the block pair. This returns the sum
501 /// of all raw edge weights from Src to Dst.
497502 uint32_t BranchProbabilityInfo::
498503 getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
499 Edge E(Src, Dst);
500 DenseMap::const_iterator I = Weights.find(E);
501
502 if (I != Weights.end())
503 return I->second;
504
505 return DEFAULT_WEIGHT;
506 }
507
504 uint32_t Weight = 0;
505 DenseMap::const_iterator MapI;
506 for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
507 if (*I == Dst) {
508 MapI = Weights.find(std::make_pair(Src, I.getSuccessorIndex()));
509 if (MapI != Weights.end())
510 Weight += MapI->second;
511 }
512 return (Weight == 0) ? DEFAULT_WEIGHT : Weight;
513 }
514
515 /// Set the edge weight for a given edge specified by PredBlock and an index
516 /// to the successors.
508517 void BranchProbabilityInfo::
509 setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst, uint32_t Weight) {
510 Weights[std::make_pair(Src, Dst)] = Weight;
518 setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors,
519 uint32_t Weight) {
520 Weights[std::make_pair(Src, IndexInSuccessors)] = Weight;
511521 DEBUG(dbgs() << "set edge " << Src->getName() << " -> "
512 << Dst->getName() << " weight to " << Weight
513 << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n"));
514 }
515
516
522 << IndexInSuccessors << " successor weight to "
523 << Weight << "\n");
524 }
525
526 /// Get an edge's probability, relative to other out-edges from Src.
527 BranchProbability BranchProbabilityInfo::
528 getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const {
529 uint32_t N = getEdgeWeight(Src, IndexInSuccessors);
530 uint32_t D = getSumForBlock(Src);
531
532 return BranchProbability(N, D);
533 }
534
535 /// Get the probability of going from Src to Dst. It returns the sum of all
536 /// probabilities for edges from Src to Dst.
517537 BranchProbability BranchProbabilityInfo::
518538 getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const {
519539
17651765 /// visitBitTestCase - this function produces one "bit test"
17661766 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
17671767 MachineBasicBlock* NextMBB,
1768 uint32_t BranchWeightToNext,
17681769 unsigned Reg,
17691770 BitTestCase &B,
17701771 MachineBasicBlock *SwitchBB) {
18021803 ISD::SETNE);
18031804 }
18041805
1805 addSuccessorWithWeight(SwitchBB, B.TargetBB);
1806 addSuccessorWithWeight(SwitchBB, NextMBB);
1806 // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
1807 addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
1808 // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
1809 addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
18071810
18081811 SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurDebugLoc(),
18091812 MVT::Other, getControlRoot(),
19261929 if (++BBI != FuncInfo.MF->end())
19271930 NextBlock = BBI;
19281931
1932 BranchProbabilityInfo *BPI = FuncInfo.BPI;
19291933 // If any two of the cases has the same destination, and if one value
19301934 // is the same as the other, but has one bit unset that the other has set,
19311935 // use bit manipulation to do two compares at once. For example:
19591963 ISD::SETEQ);
19601964
19611965 // Update successor info.
1962 addSuccessorWithWeight(SwitchBB, Small.BB);
1963 addSuccessorWithWeight(SwitchBB, Default);
1966 // Both Small and Big will jump to Small.BB, so we sum up the weights.
1967 addSuccessorWithWeight(SwitchBB, Small.BB,
1968 Small.ExtraWeight + Big.ExtraWeight);
1969 addSuccessorWithWeight(SwitchBB, Default,
1970 // The default destination is the first successor in IR.
1971 BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
19641972
19651973 // Insert the true branch.
19661974 SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
19781986 }
19791987
19801988 // Order cases by weight so the most likely case will be checked first.
1981 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1989 uint32_t UnhandledWeights = 0;
19821990 if (BPI) {
19831991 for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
1984 uint32_t IWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
1985 I->BB->getBasicBlock());
1992 uint32_t IWeight = I->ExtraWeight;
1993 UnhandledWeights += IWeight;
19861994 for (CaseItr J = CR.Range.first; J < I; ++J) {
1987 uint32_t JWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
1988 J->BB->getBasicBlock());
1995 uint32_t JWeight = J->ExtraWeight;
19891996 if (IWeight > JWeight)
19901997 std::swap(*I, *J);
19911998 }
20342041 LHS = I->Low; MHS = SV; RHS = I->High;
20352042 }
20362043
2037 uint32_t ExtraWeight = I->ExtraWeight;
2044 // The false weight should be sum of all un-handled cases.
2045 UnhandledWeights -= I->ExtraWeight;
20382046 CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
20392047 /* me */ CurBlock,
2040 /* trueweight */ ExtraWeight / 2, /* falseweight */ ExtraWeight / 2);
2048 /* trueweight */ I->ExtraWeight,
2049 /* falseweight */ UnhandledWeights);
20412050
20422051 // If emitting the first comparison, just call visitSwitchCase to emit the
20432052 // code into the current block. Otherwise, push the CaseBlock onto the
21372146 }
21382147 }
21392148
2149 // Calculate weight for each unique destination in CR.
2150 DenseMap DestWeights;
2151 if (FuncInfo.BPI)
2152 for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
2153 DenseMap::iterator Itr =
2154 DestWeights.find(I->BB);
2155 if (Itr != DestWeights.end())
2156 Itr->second += I->ExtraWeight;
2157 else
2158 DestWeights[I->BB] = I->ExtraWeight;
2159 }
2160
21402161 // Update successor info. Add one edge to each unique successor.
21412162 BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
21422163 for (std::vector::iterator I = DestBBs.begin(),
21432164 E = DestBBs.end(); I != E; ++I) {
21442165 if (!SuccsHandled[(*I)->getNumber()]) {
21452166 SuccsHandled[(*I)->getNumber()] = true;
2146 addSuccessorWithWeight(JumpTableBB, *I);
2167 DenseMap::iterator Itr =
2168 DestWeights.find(*I);
2169 addSuccessorWithWeight(JumpTableBB, *I,
2170 Itr != DestWeights.end() ? Itr->second : 0);
21472171 }
21482172 }
21492173
23742398
23752399 if (i == count) {
23762400 assert((count < 3) && "Too much destinations to test!");
2377 CasesBits.push_back(CaseBits(0, Dest, 0));
2401 CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
23782402 count++;
23792403 }
23802404
23832407
23842408 uint64_t lo = (lowValue - lowBound).getZExtValue();
23852409 uint64_t hi = (highValue - lowBound).getZExtValue();
2410 CasesBits[i].ExtraWeight += I->ExtraWeight;
23862411
23872412 for (uint64_t j = lo; j <= hi; j++) {
23882413 CasesBits[i].Mask |= 1ULL << j;
24102435 CurMF->insert(BBI, CaseBB);
24112436 BTC.push_back(BitTestCase(CasesBits[i].Mask,
24122437 CaseBB,
2413 CasesBits[i].BB));
2438 CasesBits[i].BB, CasesBits[i].ExtraWeight));
24142439
24152440 // Put SV in a virtual register to make it available from the new blocks.
24162441 ExportFromCurrentBlock(SV);
24382463
24392464 Clusterifier TheClusterifier;
24402465
2466 BranchProbabilityInfo *BPI = FuncInfo.BPI;
24412467 // Start with "simple" cases
24422468 for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
24432469 i != e; ++i) {
24442470 const BasicBlock *SuccBB = i.getCaseSuccessor();
24452471 MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
24462472
2447 TheClusterifier.add(i.getCaseValueEx(), SMBB);
2473 TheClusterifier.add(i.getCaseValueEx(), SMBB,
2474 BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0);
24482475 }
24492476
24502477 TheClusterifier.optimize();
24512478
2452 BranchProbabilityInfo *BPI = FuncInfo.BPI;
24532479 size_t numCmps = 0;
24542480 for (Clusterifier::RangeIterator i = TheClusterifier.begin(),
24552481 e = TheClusterifier.end(); i != e; ++i, ++numCmps) {
24562482 Clusterifier::Cluster &C = *i;
2457 unsigned W = 0;
2458 if (BPI) {
2459 W = BPI->getEdgeWeight(SI.getParent(), C.second->getBasicBlock());
2460 if (!W)
2461 W = 16;
2462 W *= C.first.Weight;
2463 BPI->setEdgeWeight(SI.getParent(), C.second->getBasicBlock(), W);
2464 }
2483 // Update edge weight for the cluster.
2484 unsigned W = C.first.Weight;
24652485
24662486 // FIXME: Currently work with ConstantInt based numbers.
24672487 // Changing it to APInt based is a pretty heavy for this commit.
149149 uint64_t Mask;
150150 MachineBasicBlock* BB;
151151 unsigned Bits;
152
153 CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits):
154 Mask(mask), BB(bb), Bits(bits) { }
152 uint32_t ExtraWeight;
153
154 CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits,
155 uint32_t Weight):
156 Mask(mask), BB(bb), Bits(bits), ExtraWeight(Weight) { }
155157 };
156158
157159 typedef std::vector CaseVector;
246248 typedef std::pair JumpTableBlock;
247249
248250 struct BitTestCase {
249 BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr):
250 Mask(M), ThisBB(T), TargetBB(Tr) { }
251 BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr,
252 uint32_t Weight):
253 Mask(M), ThisBB(T), TargetBB(Tr), ExtraWeight(Weight) { }
251254 uint64_t Mask;
252255 MachineBasicBlock *ThisBB;
253256 MachineBasicBlock *TargetBB;
257 uint32_t ExtraWeight;
254258 };
255259
256260 typedef SmallVector BitTestInfo;
451455 void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB);
452456 void visitBitTestCase(BitTestBlock &BB,
453457 MachineBasicBlock* NextMBB,
458 uint32_t BranchWeightToNext,
454459 unsigned Reg,
455460 BitTestCase &B,
456461 MachineBasicBlock *SwitchBB);
12081208 CodeGenAndEmitDAG();
12091209 }
12101210
1211 uint32_t UnhandledWeight = 0;
1212 for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j)
1213 UnhandledWeight += SDB->BitTestCases[i].Cases[j].ExtraWeight;
1214
12111215 for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) {
1216 UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight;
12121217 // Set the current basic block to the mbb we wish to insert the code into
12131218 FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB;
12141219 FuncInfo->InsertPt = FuncInfo->MBB->end();
12161221 if (j+1 != ej)
12171222 SDB->visitBitTestCase(SDB->BitTestCases[i],
12181223 SDB->BitTestCases[i].Cases[j+1].ThisBB,
1224 UnhandledWeight,
12191225 SDB->BitTestCases[i].Reg,
12201226 SDB->BitTestCases[i].Cases[j],
12211227 FuncInfo->MBB);
12221228 else
12231229 SDB->visitBitTestCase(SDB->BitTestCases[i],
12241230 SDB->BitTestCases[i].Default,
1231 UnhandledWeight,
12251232 SDB->BitTestCases[i].Reg,
12261233 SDB->BitTestCases[i].Cases[j],
12271234 FuncInfo->MBB);
8787 }
8888
8989 !1 = metadata !{metadata !"branch_weights", i32 4, i32 4, i32 64, i32 4, i32 4}
90
91 define i32 @test4(i32 %x) nounwind uwtable readnone ssp {
92 ; CHECK: Printing analysis {{.*}} for function 'test4'
93 entry:
94 %conv = sext i32 %x to i64
95 switch i64 %conv, label %return [
96 i64 0, label %sw.bb
97 i64 1, label %sw.bb
98 i64 2, label %sw.bb
99 i64 5, label %sw.bb1
100 ], !prof !2
101 ; CHECK: edge entry -> return probability is 7 / 85
102 ; CHECK: edge entry -> sw.bb probability is 14 / 85
103 ; CHECK: edge entry -> sw.bb1 probability is 64 / 85
104
105 sw.bb:
106 br label %return
107
108 sw.bb1:
109 br label %return
110
111 return:
112 %retval.0 = phi i32 [ 5, %sw.bb1 ], [ 1, %sw.bb ], [ 0, %entry ]
113 ret i32 %retval.0
114 }
115
116 !2 = metadata !{metadata !"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}
0 ; RUN: llc < %s -print-machineinstrs=expand-isel-pseudos -o /dev/null 2>&1 | FileCheck %s
1
2 ; Make sure we have the correct weight attached to each successor.
3 define i32 @test2(i32 %x) nounwind uwtable readnone ssp {
4 ; CHECK: Machine code for function test2:
5 entry:
6 %conv = sext i32 %x to i64
7 switch i64 %conv, label %return [
8 i64 0, label %sw.bb
9 i64 1, label %sw.bb
10 i64 4, label %sw.bb
11 i64 5, label %sw.bb1
12 ], !prof !0
13 ; CHECK: BB#0: derived from LLVM BB %entry
14 ; CHECK: Successors according to CFG: BB#2(64) BB#4(14)
15 ; CHECK: BB#4: derived from LLVM BB %entry
16 ; CHECK: Successors according to CFG: BB#1(10) BB#5(4)
17 ; CHECK: BB#5: derived from LLVM BB %entry
18 ; CHECK: Successors according to CFG: BB#1(4) BB#3(7)
19
20 sw.bb:
21 br label %return
22
23 sw.bb1:
24 br label %return
25
26 return:
27 %retval.0 = phi i32 [ 5, %sw.bb1 ], [ 1, %sw.bb ], [ 0, %entry ]
28 ret i32 %retval.0
29 }
30
31 !0 = metadata !{metadata !"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}