llvm.org GIT mirror llvm / 8e83fe2
Revert r254348: "Replace all weight-based interfaces in MBB with probability-based interfaces, and update all uses of old interfaces." and the follow-up r254356: "Fix a bug in MachineBlockPlacement that may cause assertion failure during BranchProbability construction." Asserts were firing in Chromium builds. See PR25687. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@254366 91177308-0d34-0410-b5e6-96231b3b80d8 Hans Wennborg 4 years ago
34 changed file(s) with 429 addition(s) and 305 deletion(s). Raw diff Collapse all Expand all
5959 /// It returns the sum of all probabilities for edges from Src to Dst.
6060 BranchProbability getEdgeProbability(const BasicBlock *Src,
6161 const BasicBlock *Dst) const;
62
63 BranchProbability getEdgeProbability(const BasicBlock *Src,
64 succ_const_iterator Dst) const;
6562
6663 /// \brief Test if an edge is hot relative to other out-edges of the Src.
6764 ///
8989 /// Keep track of the predecessor / successor basic blocks.
9090 std::vector Predecessors;
9191 std::vector Successors;
92
93 /// Keep track of the weights to the successors. This vector has the same
94 /// order as Successors, or it is empty if we don't use it (disable
95 /// optimization).
96 std::vector Weights;
97 typedef std::vector::iterator weight_iterator;
98 typedef std::vector::const_iterator const_weight_iterator;
9299
93100 /// Keep track of the probabilities to the successors. This vector has the
94101 /// same order as Successors, or it is empty if we don't use it (disable
433440 // Machine-CFG mutators
434441
435442 /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list
443 /// of Succ is automatically updated. WEIGHT parameter is stored in Weights
444 /// list and it may be used by MachineBranchProbabilityInfo analysis to
445 /// calculate branch probability.
446 ///
447 /// Note that duplicate Machine CFG edges are not allowed.
448 void addSuccessor(MachineBasicBlock *Succ, uint32_t Weight = 0);
449
450 /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list
451 /// of Succ is automatically updated. The weight is not provided because BPI
452 /// is not available (e.g. -O0 is used), in which case edge weights won't be
453 /// used. Using this interface can save some space.
454 void addSuccessorWithoutWeight(MachineBasicBlock *Succ);
455
456 /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list
436457 /// of Succ is automatically updated. PROB parameter is stored in
437 /// Probabilities list. The default probability is set as unknown. Mixing
438 /// known and unknown probabilities in successor list is not allowed. When all
439 /// successors have unknown probabilities, 1 / N is returned as the
440 /// probability for each successor, where N is the number of successors.
458 /// Probabilities list.
441459 ///
442460 /// Note that duplicate Machine CFG edges are not allowed.
443 void addSuccessor(MachineBasicBlock *Succ,
444 BranchProbability Prob = BranchProbability::getUnknown());
461 void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob);
445462
446463 /// Add Succ as a successor of this MachineBasicBlock. The Predecessors list
447464 /// of Succ is automatically updated. The probability is not provided because
449466 /// won't be used. Using this interface can save some space.
450467 void addSuccessorWithoutProb(MachineBasicBlock *Succ);
451468
469 /// Set successor weight of a given iterator.
470 void setSuccWeight(succ_iterator I, uint32_t Weight);
471
452472 /// Set successor probability of a given iterator.
453473 void setSuccProbability(succ_iterator I, BranchProbability Prob);
454474
467487 /// Return the iterator to the element after the one removed.
468488 succ_iterator removeSuccessor(succ_iterator I);
469489
470 /// Replace successor OLD with NEW and update probability info.
490 /// Replace successor OLD with NEW and update weight info.
471491 void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New);
472492
473493 /// Transfers all the successors from MBB to this machine basic block (i.e.,
478498 /// Transfers all the successors, as in transferSuccessors, and update PHI
479499 /// operands in the successor blocks which refer to FromMBB to refer to this.
480500 void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB);
501
502 /// Return true if any of the successors have weights attached to them.
503 bool hasSuccessorWeights() const { return !Weights.empty(); }
481504
482505 /// Return true if any of the successors have probabilities attached to them.
483506 bool hasSuccessorProbabilities() const { return !Probs.empty(); }
735758
736759
737760 private:
761 /// Return weight iterator corresponding to the I successor iterator.
762 weight_iterator getWeightIterator(succ_iterator I);
763 const_weight_iterator getWeightIterator(const_succ_iterator I) const;
764
738765 /// Return probability iterator corresponding to the I successor iterator.
739766 probability_iterator getProbabilityIterator(succ_iterator I);
740767 const_probability_iterator
742769
743770 friend class MachineBranchProbabilityInfo;
744771 friend class MIPrinter;
772
773 /// Return weight of the edge from this block to MBB. This method should NOT
774 /// be called directly, but by using getEdgeWeight method from
775 /// MachineBranchProbabilityInfo class.
776 uint32_t getSuccWeight(const_succ_iterator Succ) const;
745777
746778 /// Return probability of the edge from this block to MBB. This method should
747779 /// NOT be called directly, but by using getEdgeProbability method from
5454 uint32_t getEdgeWeight(const MachineBasicBlock *Src,
5555 MachineBasicBlock::const_succ_iterator Dst) const;
5656
57 // Return edge probability.
58 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
59 const MachineBasicBlock *Dst) const;
60
61 // Same as above, but using a const_succ_iterator from Src. This is faster
62 // when the iterator is already available.
63 BranchProbability
64 getEdgeProbability(const MachineBasicBlock *Src,
65 MachineBasicBlock::const_succ_iterator Dst) const;
57 // Get sum of the block successors' weights, potentially scaling them to fit
58 // within 32-bits. If scaling is required, sets Scale based on the necessary
59 // adjustment. Any edge weights used with the sum should be divided by Scale.
60 uint32_t getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const;
6661
6762 // A 'Hot' edge is an edge which probability is >= 80%.
6863 bool isEdgeHot(const MachineBasicBlock *Src,
7166 // Return a hot successor for the block BB or null if there isn't one.
7267 // NB: This routine's complexity is linear on the number of successors.
7368 MachineBasicBlock *getHotSucc(MachineBasicBlock *MBB) const;
69
70 // Return a probability as a fraction between 0 (0% probability) and
71 // 1 (100% probability), however the value is never equal to 0, and can be 1
72 // only iff SRC block has only one successor.
73 // NB: This routine's complexity is linear on the number of successors of
74 // Src. Querying sequentially for each successor's probability is a quadratic
75 // query pattern.
76 BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
77 const MachineBasicBlock *Dst) const;
7478
7579 // Print value between 0 (0% probability) and 1 (100% probability),
7680 // however the value is never equal to 0, and can be 1 only iff SRC block
5252 // Create a BranchProbability object with the given numerator and 1<<31
5353 // as denominator.
5454 static BranchProbability getRaw(uint32_t N) { return BranchProbability(N); }
55 // Create a BranchProbability object from 64-bit integers.
56 static BranchProbability getBranchProbability(uint64_t Numerator,
57 uint64_t Denominator);
5855
5956 // Normalize given probabilties so that the sum of them becomes approximate
6057 // one.
133130
134131 bool operator==(BranchProbability RHS) const { return N == RHS.N; }
135132 bool operator!=(BranchProbability RHS) const { return !(*this == RHS); }
136
137 bool operator<(BranchProbability RHS) const {
138 assert(N != UnknownN && RHS.N != UnknownN &&
139 "Unknown probability cannot participate in comparisons.");
140 return N < RHS.N;
141 }
142
143 bool operator>(BranchProbability RHS) const {
144 assert(N != UnknownN && RHS.N != UnknownN &&
145 "Unknown probability cannot participate in comparisons.");
146 return RHS < *this;
147 }
148
149 bool operator<=(BranchProbability RHS) const {
150 assert(N != UnknownN && RHS.N != UnknownN &&
151 "Unknown probability cannot participate in comparisons.");
152 return !(RHS < *this);
153 }
154
155 bool operator>=(BranchProbability RHS) const {
156 assert(N != UnknownN && RHS.N != UnknownN &&
157 "Unknown probability cannot participate in comparisons.");
158 return !(*this < RHS);
159 }
133 bool operator<(BranchProbability RHS) const { return N < RHS.N; }
134 bool operator>(BranchProbability RHS) const { return RHS < *this; }
135 bool operator<=(BranchProbability RHS) const { return !(RHS < *this); }
136 bool operator>=(BranchProbability RHS) const { return !(*this < RHS); }
160137 };
161138
162139 inline raw_ostream &operator<<(raw_ostream &OS, BranchProbability Prob) {
646646 return BranchProbability(N, D);
647647 }
648648
649 BranchProbability
650 BranchProbabilityInfo::getEdgeProbability(const BasicBlock *Src,
651 succ_const_iterator Dst) const {
652 return getEdgeProbability(Src, Dst.getSuccessorIndex());
653 }
654
655649 raw_ostream &
656650 BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS,
657651 const BasicBlock *Src,
10981098 if (TailMBB.succ_size() <= 1)
10991099 return;
11001100
1101 auto SumEdgeFreq =
1102 std::accumulate(EdgeFreqLs.begin(), EdgeFreqLs.end(), BlockFrequency(0))
1103 .getFrequency();
1101 auto MaxEdgeFreq = *std::max_element(EdgeFreqLs.begin(), EdgeFreqLs.end());
1102 uint64_t Scale = MaxEdgeFreq.getFrequency() / UINT32_MAX + 1;
11041103 auto EdgeFreq = EdgeFreqLs.begin();
11051104
11061105 for (auto SuccI = TailMBB.succ_begin(), SuccE = TailMBB.succ_end();
11071106 SuccI != SuccE; ++SuccI, ++EdgeFreq)
1108 TailMBB.setSuccProbability(
1109 SuccI, BranchProbability::getBranchProbability(EdgeFreq->getFrequency(),
1110 SumEdgeFreq));
1107 TailMBB.setSuccWeight(SuccI, EdgeFreq->getFrequency() / Scale);
11111108 }
11121109
11131110 //===----------------------------------------------------------------------===//
3131 #include "llvm/Target/TargetLowering.h"
3232 #include "llvm/Target/TargetRegisterInfo.h"
3333 #include "llvm/Target/TargetSubtargetInfo.h"
34 #include
3534
3635 using namespace llvm;
3736
11511150 return true;
11521151 }
11531152
1153 /// Scale down weights to fit into uint32_t. NewTrue is the new weight
1154 /// for successor TrueBB, and NewFalse is the new weight for successor
1155 /// FalseBB.
1156 static void ScaleWeights(uint64_t NewTrue, uint64_t NewFalse,
1157 MachineBasicBlock *MBB,
1158 const MachineBasicBlock *TrueBB,
1159 const MachineBasicBlock *FalseBB,
1160 const MachineBranchProbabilityInfo *MBPI) {
1161 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
1162 uint32_t Scale = (NewMax / UINT32_MAX) + 1;
1163 for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
1164 SE = MBB->succ_end();
1165 SI != SE; ++SI) {
1166 if (*SI == TrueBB)
1167 MBB->setSuccWeight(SI, (uint32_t)(NewTrue / Scale));
1168 else if (*SI == FalseBB)
1169 MBB->setSuccWeight(SI, (uint32_t)(NewFalse / Scale));
1170 else
1171 MBB->setSuccWeight(SI, MBPI->getEdgeWeight(MBB, SI) / Scale);
1172 }
1173 }
1174
11541175 /// IfConvertTriangle - If convert a triangle sub-CFG.
11551176 ///
11561177 bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
12071228 DontKill.clear();
12081229
12091230 bool HasEarlyExit = CvtBBI->FalseBB != nullptr;
1210 BranchProbability CvtNext, CvtFalse, BBNext, BBCvt;
1231 uint64_t CvtNext = 0, CvtFalse = 0, BBNext = 0, BBCvt = 0, SumWeight = 0;
1232 uint32_t WeightScale = 0;
12111233
12121234 if (HasEarlyExit) {
1213 // Get probabilities before modifying CvtBBI->BB and BBI.BB.
1214 CvtNext = MBPI->getEdgeProbability(CvtBBI->BB, NextBBI->BB);
1215 CvtFalse = MBPI->getEdgeProbability(CvtBBI->BB, CvtBBI->FalseBB);
1216 BBNext = MBPI->getEdgeProbability(BBI.BB, NextBBI->BB);
1217 BBCvt = MBPI->getEdgeProbability(BBI.BB, CvtBBI->BB);
1235 // Get weights before modifying CvtBBI->BB and BBI.BB.
1236 CvtNext = MBPI->getEdgeWeight(CvtBBI->BB, NextBBI->BB);
1237 CvtFalse = MBPI->getEdgeWeight(CvtBBI->BB, CvtBBI->FalseBB);
1238 BBNext = MBPI->getEdgeWeight(BBI.BB, NextBBI->BB);
1239 BBCvt = MBPI->getEdgeWeight(BBI.BB, CvtBBI->BB);
1240 SumWeight = MBPI->getSumForBlock(CvtBBI->BB, WeightScale);
12181241 }
12191242
12201243 if (CvtBBI->BB->pred_size() > 1) {
12421265 CvtBBI->BrCond.end());
12431266 if (TII->ReverseBranchCondition(RevCond))
12441267 llvm_unreachable("Unable to reverse branch condition!");
1245
1246 // Update the edge probability for both CvtBBI->FalseBB and NextBBI.
1247 // NewNext = New_Prob(BBI.BB, NextBBI->BB) =
1248 // Prob(BBI.BB, NextBBI->BB) +
1249 // Prob(BBI.BB, CvtBBI->BB) * Prob(CvtBBI->BB, NextBBI->BB)
1250 // NewFalse = New_Prob(BBI.BB, CvtBBI->FalseBB) =
1251 // Prob(BBI.BB, CvtBBI->BB) * Prob(CvtBBI->BB, CvtBBI->FalseBB)
1252 auto NewTrueBB = getNextBlock(BBI.BB);
1253 auto NewNext = BBNext + BBCvt * CvtNext;
1254 auto NewTrueBBIter =
1255 std::find(BBI.BB->succ_begin(), BBI.BB->succ_end(), NewTrueBB);
1256 assert(NewTrueBBIter != BBI.BB->succ_end() &&
1257 "NewTrueBB is not a successor of BBI.BB.");
1258 BBI.BB->setSuccProbability(NewTrueBBIter, NewNext);
1259
1260 auto NewFalse = BBCvt * CvtFalse;
12611268 TII->InsertBranch(*BBI.BB, CvtBBI->FalseBB, nullptr, RevCond, dl);
1262 BBI.BB->addSuccessor(CvtBBI->FalseBB, NewFalse);
1269 BBI.BB->addSuccessor(CvtBBI->FalseBB);
1270 // Update the edge weight for both CvtBBI->FalseBB and NextBBI.
1271 // New_Weight(BBI.BB, NextBBI->BB) =
1272 // Weight(BBI.BB, NextBBI->BB) * getSumForBlock(CvtBBI->BB) +
1273 // Weight(BBI.BB, CvtBBI->BB) * Weight(CvtBBI->BB, NextBBI->BB)
1274 // New_Weight(BBI.BB, CvtBBI->FalseBB) =
1275 // Weight(BBI.BB, CvtBBI->BB) * Weight(CvtBBI->BB, CvtBBI->FalseBB)
1276
1277 uint64_t NewNext = BBNext * SumWeight + (BBCvt * CvtNext) / WeightScale;
1278 uint64_t NewFalse = (BBCvt * CvtFalse) / WeightScale;
1279 // We need to scale down all weights of BBI.BB to fit uint32_t.
1280 // Here BBI.BB is connected to CvtBBI->FalseBB and will fall through to
1281 // the next block.
1282 ScaleWeights(NewNext, NewFalse, BBI.BB, getNextBlock(BBI.BB),
1283 CvtBBI->FalseBB, MBPI);
12631284 }
12641285
12651286 // Merge in the 'false' block if the 'false' block has no other
15021523 MergeBlocks(BBI, TailBBI);
15031524 TailBBI.IsDone = true;
15041525 } else {
1505 BBI.BB->addSuccessor(TailBB, BranchProbability::getOne());
1526 BBI.BB->addSuccessor(TailBB);
15061527 InsertUncondBranch(BBI.BB, TailBB, TII);
15071528 BBI.HasFallThrough = false;
15081529 }
16661687 FromBBI.BB->succ_end());
16671688 MachineBasicBlock *NBB = getNextBlock(FromBBI.BB);
16681689 MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr;
1669 // The edge probability from ToBBI.BB to FromBBI.BB, which is only needed when
1690
1691 // The edge weight from ToBBI.BB to FromBBI.BB, which is only needed when
16701692 // AddEdges is true and FromBBI.BB is a successor of ToBBI.BB.
1671 auto To2FromProb = BranchProbability::getZero();
1693 uint32_t To2FromWeight = 0;
1694 // WeightScale and SumWeight are for calculating successor probabilities of
1695 // FromBBI.BB.
1696 uint32_t WeightScale = 0;
1697 uint32_t SumWeight = 0;
16721698 if (AddEdges && ToBBI.BB->isSuccessor(FromBBI.BB)) {
1673 To2FromProb = MBPI->getEdgeProbability(ToBBI.BB, FromBBI.BB);
1674 // Set the edge probability from ToBBI.BB to FromBBI.BB to zero to avoid the
1675 // edge probability being merged to other edges when this edge is removed
1676 // later.
1677 ToBBI.BB->setSuccProbability(
1678 std::find(ToBBI.BB->succ_begin(), ToBBI.BB->succ_end(), FromBBI.BB),
1679 BranchProbability::getZero());
1680 }
1681
1682 if (AddEdges && ToBBI.BB->isSuccessor(FromBBI.BB)) {
1683 // Set the edge probability from ToBBI.BB to FromBBI.BB to zero to avoid the
1684 // edge probability being merged to other edges when this edge is removed
1685 // later.
1686 ToBBI.BB->setSuccProbability(
1687 std::find(ToBBI.BB->succ_begin(), ToBBI.BB->succ_end(), FromBBI.BB),
1688 BranchProbability::getZero());
1699 To2FromWeight = MBPI->getEdgeWeight(ToBBI.BB, FromBBI.BB);
1700 // Set the edge weight from ToBBI.BB to FromBBI.BB to zero to avoid the edge
1701 // weight being merged to other edges when this edge is removed later.
1702 ToBBI.BB->setSuccWeight(
1703 std::find(ToBBI.BB->succ_begin(), ToBBI.BB->succ_end(), FromBBI.BB), 0);
1704 SumWeight = MBPI->getSumForBlock(FromBBI.BB, WeightScale);
16891705 }
16901706
16911707 for (unsigned i = 0, e = FromSuccs.size(); i != e; ++i) {
16941710 if (Succ == FallThrough)
16951711 continue;
16961712
1697 auto NewProb = BranchProbability::getZero();
1713 uint32_t NewWeight = 0;
16981714 if (AddEdges) {
1699 // Calculate the edge probability for the edge from ToBBI.BB to Succ,
1700 // which is a portion of the edge probability from FromBBI.BB to Succ. The
1701 // portion ratio is the edge probability from ToBBI.BB to FromBBI.BB (if
1702 // FromBBI is a successor of ToBBI.BB. See comment below for excepion).
1703 NewProb = MBPI->getEdgeProbability(FromBBI.BB, Succ);
1704
1705 // To2FromProb is 0 when FromBBI.BB is not a successor of ToBBI.BB. This
1715 // Calculate the edge weight for the edge from ToBBI.BB to Succ, which is
1716 // a portion of the edge weight from FromBBI.BB to Succ. The portion ratio
1717 // is the edge probability from ToBBI.BB to FromBBI.BB (if FromBBI is a
1718 // successor of ToBBI.BB. See comment below for excepion).
1719 NewWeight = MBPI->getEdgeWeight(FromBBI.BB, Succ);
1720
1721 // To2FromWeight is 0 when FromBBI.BB is not a successor of ToBBI.BB. This
17061722 // only happens when if-converting a diamond CFG and FromBBI.BB is the
17071723 // tail BB. In this case FromBBI.BB post-dominates ToBBI.BB and hence we
1708 // could just use the probabilities on FromBBI.BB's out-edges when adding
1709 // new successors.
1710 if (!To2FromProb.isZero())
1711 NewProb *= To2FromProb;
1724 // could just use the weights on FromBBI.BB's out-edges when adding new
1725 // successors.
1726 if (To2FromWeight > 0) {
1727 BranchProbability Prob(NewWeight / WeightScale, SumWeight);
1728 NewWeight = Prob.scale(To2FromWeight);
1729 }
17121730 }
17131731
17141732 FromBBI.BB->removeSuccessor(Succ);
17151733
17161734 if (AddEdges) {
1717 // If the edge from ToBBI.BB to Succ already exists, update the
1718 // probability of this edge by adding NewWeight to it. An example is shown
1719 // below, in which A is ToBBI.BB and B is FromBBI.BB. In this case we
1720 // don't have to set C as A's successor as it already is. We only need to
1721 // update the edge probability on A->C. Note that B will not be
1722 // immediately removed from A's successors. It is possible that B->D is
1723 // not removed either if D is a fallthrough of B. Later the edge A->D
1724 // (generated here) and B->D will be combined into one edge. To maintain
1725 // correct edge probability of this combined edge, we need to set the edge
1726 // probability of A->B to zero, which is already done above. The edge
1727 // probability on A->D is calculated by scaling the original probability
1728 // on A->B by the probability of B->D.
1735 // If the edge from ToBBI.BB to Succ already exists, update the weight of
1736 // this edge by adding NewWeight to it. An example is shown below, in
1737 // which A is ToBBI.BB and B is FromBBI.BB. In this case we don't have to
1738 // set C as A's successor as it already is. We only need to update the
1739 // edge weight on A->C. Note that B will not be immediately removed from
1740 // A's successors. It is possible that B->D is not removed either if D is
1741 // a fallthrough of B. Later the edge A->D (generated here) and B->D will
1742 // be combined into one edge. To maintain correct edge weight of this
1743 // combined edge, we need to set the edge weight of A->B to zero, which is
1744 // already done above. The edge weight on A->D is calculated by scaling
1745 // the original weight on A->B by the probability of B->D.
17291746 //
17301747 // Before ifcvt: After ifcvt (assume B->D is kept):
17311748 //
17371754 // C D C D
17381755 //
17391756 if (ToBBI.BB->isSuccessor(Succ))
1740 ToBBI.BB->setSuccProbability(
1757 ToBBI.BB->setSuccWeight(
17411758 std::find(ToBBI.BB->succ_begin(), ToBBI.BB->succ_end(), Succ),
1742 MBPI->getEdgeProbability(ToBBI.BB, Succ) + NewProb);
1759 MBPI->getEdgeWeight(ToBBI.BB, Succ) + NewWeight);
17431760 else
1744 ToBBI.BB->addSuccessor(Succ, NewProb);
1761 ToBBI.BB->addSuccessor(Succ, NewWeight);
17451762 }
17461763 }
17471764
458458 if (expectAndConsume(MIToken::rparen))
459459 return true;
460460 }
461 MBB.addSuccessor(SuccMBB, BranchProbability::getRaw(Weight));
461 MBB.addSuccessor(SuccMBB, Weight);
462462 } while (consumeIfPresent(MIToken::comma));
463 MBB.normalizeSuccProbs();
464463 return false;
465464 }
466465
460460 if (I != MBB.succ_begin())
461461 OS << ", ";
462462 printMBBReference(**I);
463 if (MBB.hasSuccessorProbabilities())
464 OS << '(' << MBB.getSuccProbability(I) << ')';
463 if (MBB.hasSuccessorWeights())
464 OS << '(' << MBB.getSuccWeight(I) << ')';
465465 }
466466 OS << "\n";
467467 HasLineAttributes = true;
318318 OS << " Successors according to CFG:";
319319 for (const_succ_iterator SI = succ_begin(), E = succ_end(); SI != E; ++SI) {
320320 OS << " BB#" << (*SI)->getNumber();
321 if (!Probs.empty())
322 OS << '(' << *getProbabilityIterator(SI) << ')';
321 if (!Weights.empty())
322 OS << '(' << *getWeightIterator(SI) << ')';
323323 }
324324 OS << '\n';
325325 }
505505 }
506506 }
507507
508 void MachineBasicBlock::addSuccessor(MachineBasicBlock *Succ, uint32_t Weight) {
509 // Weight list is either empty (if successor list isn't empty, this means
510 // disabled optimization) or has the same size as successor list.
511 if (!(Weights.empty() && !Successors.empty()))
512 Weights.push_back(Weight);
513 Successors.push_back(Succ);
514 Succ->addPredecessor(this);
515 }
516
517 void MachineBasicBlock::addSuccessorWithoutWeight(MachineBasicBlock *Succ) {
518 // We need to make sure weight list is either empty or has the same size of
519 // successor list. When this function is called, we can safely delete all
520 // weight in the list.
521 Weights.clear();
522 Successors.push_back(Succ);
523 Succ->addPredecessor(this);
524 }
525
508526 void MachineBasicBlock::addSuccessor(MachineBasicBlock *Succ,
509527 BranchProbability Prob) {
510528 // Probability list is either empty (if successor list isn't empty, this means
511529 // disabled optimization) or has the same size as successor list.
512530 if (!(Probs.empty() && !Successors.empty())) {
513 assert((Probs.empty() || (Prob.isUnknown() && Probs.back().isUnknown()) ||
514 (!Prob.isUnknown() && !Probs.back().isUnknown())) &&
515 "Successors with both known and unknwon probabilities are not "
516 "allowed.");
517531 Probs.push_back(Prob);
532 // FIXME: Temporarily use the numerator of the probability to represent edge
533 // weight. This will be removed once all weight-version interfaces in MBB
534 // are replaced with probability-version interfaces.
535 Weights.push_back(Prob.getNumerator());
518536 }
519537 Successors.push_back(Succ);
520538 Succ->addPredecessor(this);
525543 // of successor list. When this function is called, we can safely delete all
526544 // probability in the list.
527545 Probs.clear();
546 Weights.clear();
528547 Successors.push_back(Succ);
529548 Succ->addPredecessor(this);
530549 }
538557 MachineBasicBlock::removeSuccessor(succ_iterator I) {
539558 assert(I != Successors.end() && "Not a current successor!");
540559
560 // If Weight list is empty it means we don't use it (disabled optimization).
561 if (!Weights.empty()) {
562 weight_iterator WI = getWeightIterator(I);
563 Weights.erase(WI);
564 }
565
566 // FIXME: Temporarily comment the following code as probabilities are now only
567 // used during instruction lowering, but this interface is called in later
568 // passes. Uncomment it once all edge weights are replaced with probabilities.
569 #if 0
541570 // If probability list is empty it means we don't use it (disabled
542571 // optimization).
543572 if (!Probs.empty()) {
544573 probability_iterator WI = getProbabilityIterator(I);
545574 Probs.erase(WI);
546575 }
576 #endif
547577
548578 (*I)->removePredecessor(this);
549579 return Successors.erase(I);
580610 }
581611
582612 // New is already a successor.
613 // Update its weight instead of adding a duplicate edge.
614 if (!Weights.empty())
615 *getWeightIterator(NewI) += *getWeightIterator(OldI);
616 // FIXME: Temporarily comment the following code as probabilities are now only
617 // used during instruction lowering, but this interface is called in later
618 // passes. Uncomment it once all edge weights are replaced with probabilities.
619 #if 0
583620 // Update its probability instead of adding a duplicate edge.
584 if (!Probs.empty()) {
585 auto ProbIter = getProbabilityIterator(NewI);
586 if (!ProbIter->isUnknown())
587 *ProbIter += *getProbabilityIterator(OldI);
588 }
621 if (!Probs.empty())
622 *getProbabilityIterator(NewI) += *getProbabilityIterator(OldI);
623 #endif
589624 removeSuccessor(OldI);
590625 }
591626
605640
606641 while (!FromMBB->succ_empty()) {
607642 MachineBasicBlock *Succ = *FromMBB->succ_begin();
608
609 // If probability list is empty it means we don't use it (disabled optimization).
610 if (!FromMBB->Probs.empty()) {
611 auto Prob = *FromMBB->Probs.begin();
612 addSuccessor(Succ, Prob);
613 } else
614 addSuccessorWithoutProb(Succ);
615
643 uint32_t Weight = 0;
644
645 // If Weight list is empty it means we don't use it (disabled optimization).
646 if (!FromMBB->Weights.empty())
647 Weight = *FromMBB->Weights.begin();
648
649 addSuccessor(Succ, Weight);
616650 FromMBB->removeSuccessor(Succ);
617651 }
618652 }
624658
625659 while (!FromMBB->succ_empty()) {
626660 MachineBasicBlock *Succ = *FromMBB->succ_begin();
627 if (!FromMBB->Probs.empty()) {
628 auto Prob = *FromMBB->Probs.begin();
629 addSuccessor(Succ, Prob);
630 } else
631 addSuccessorWithoutProb(Succ);
661 uint32_t Weight = 0;
662 if (!FromMBB->Weights.empty())
663 Weight = *FromMBB->Weights.begin();
664 addSuccessor(Succ, Weight);
632665 FromMBB->removeSuccessor(Succ);
633666
634667 // Fix up any PHI nodes in the successor.
11121145 return DL;
11131146 }
11141147
1115 /// Return probability of the edge from this block to MBB.
1148 /// Return weight of the edge from this block to MBB.
1149 uint32_t MachineBasicBlock::getSuccWeight(const_succ_iterator Succ) const {
1150 if (Weights.empty())
1151 return 0;
1152
1153 return *getWeightIterator(Succ);
1154 }
1155
1156 /// Return probability of the edge from this block to MBB. If probability list
1157 /// is empty, return a default probability which is 1/N, where N is the number
1158 /// of successors. If the probability of the given successor is unknown, then
1159 /// sum up all known probabilities and return the complement of the sum divided
1160 /// by the number of unknown probabilities.
11161161 BranchProbability
11171162 MachineBasicBlock::getSuccProbability(const_succ_iterator Succ) const {
1118 if (Probs.empty() || Probs.back().isUnknown())
1163 if (Probs.empty())
11191164 return BranchProbability(1, succ_size());
11201165
1121 return *getProbabilityIterator(Succ);
1166 auto Prob = *getProbabilityIterator(Succ);
1167 assert(!Prob.isUnknown());
1168 return Prob;
1169 }
1170
1171 /// Set successor weight of a given iterator.
1172 void MachineBasicBlock::setSuccWeight(succ_iterator I, uint32_t Weight) {
1173 if (Weights.empty())
1174 return;
1175 *getWeightIterator(I) = Weight;
11221176 }
11231177
11241178 /// Set successor probability of a given iterator.
11251179 void MachineBasicBlock::setSuccProbability(succ_iterator I,
11261180 BranchProbability Prob) {
11271181 assert(!Prob.isUnknown());
1128 if (Probs.empty())
1182 if (Probs.empty() || Weights.empty())
11291183 return;
11301184 *getProbabilityIterator(I) = Prob;
1185 // FIXME: Temporarily use the numerator of the probability to represent edge
1186 // weight. This will be removed once all weight-version interfaces in MBB
1187 // are replaces with probability-version interfaces.
1188 *getWeightIterator(I) = Prob.getNumerator();
1189 }
1190
1191 /// Return wight iterator corresonding to the I successor iterator.
1192 MachineBasicBlock::weight_iterator MachineBasicBlock::
1193 getWeightIterator(MachineBasicBlock::succ_iterator I) {
1194 assert(Weights.size() == Successors.size() && "Async weight list!");
1195 size_t index = std::distance(Successors.begin(), I);
1196 assert(index < Weights.size() && "Not a current successor!");
1197 return Weights.begin() + index;
1198 }
1199
1200 /// Return wight iterator corresonding to the I successor iterator.
1201 MachineBasicBlock::const_weight_iterator MachineBasicBlock::
1202 getWeightIterator(MachineBasicBlock::const_succ_iterator I) const {
1203 assert(Weights.size() == Successors.size() && "Async weight list!");
1204 const size_t index = std::distance(Successors.begin(), I);
1205 assert(index < Weights.size() && "Not a current successor!");
1206 return Weights.begin() + index;
1207 }
1208
1209 /// Return probability iterator corresonding to the I successor iterator.
1210 MachineBasicBlock::probability_iterator
1211 MachineBasicBlock::getProbabilityIterator(MachineBasicBlock::succ_iterator I) {
1212 assert(Probs.size() == Successors.size() && "Async probability list!");
1213 const size_t index = std::distance(Successors.begin(), I);
1214 assert(index < Probs.size() && "Not a current successor!");
1215 return Probs.begin() + index;
11311216 }
11321217
11331218 /// Return probability iterator corresonding to the I successor iterator
11341219 MachineBasicBlock::const_probability_iterator
11351220 MachineBasicBlock::getProbabilityIterator(
11361221 MachineBasicBlock::const_succ_iterator I) const {
1137 assert(Probs.size() == Successors.size() && "Async probability list!");
1138 const size_t index = std::distance(Successors.begin(), I);
1139 assert(index < Probs.size() && "Not a current successor!");
1140 return Probs.begin() + index;
1141 }
1142
1143 /// Return probability iterator corresonding to the I successor iterator.
1144 MachineBasicBlock::probability_iterator
1145 MachineBasicBlock::getProbabilityIterator(MachineBasicBlock::succ_iterator I) {
11461222 assert(Probs.size() == Successors.size() && "Async probability list!");
11471223 const size_t index = std::distance(Successors.begin(), I);
11481224 assert(index < Probs.size() && "Not a current successor!");
379379 const BranchProbability HotProb(4, 5); // 80%
380380
381381 MachineBasicBlock *BestSucc = nullptr;
382 auto BestProb = BranchProbability::getZero();
383
384 // Adjust edge probabilities by excluding edges pointing to blocks that is
385 // either not in BlockFilter or is already in the current chain. Consider the
386 // following CFG:
382 // FIXME: Due to the performance of the probability and weight routines in
383 // the MBPI analysis, we manually compute probabilities using the edge
384 // weights. This is suboptimal as it means that the somewhat subtle
385 // definition of edge weight semantics is encoded here as well. We should
386 // improve the MBPI interface to efficiently support query patterns such as
387 // this.
388 uint32_t BestWeight = 0;
389 uint32_t WeightScale = 0;
390 uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale);
391
392 // Adjust sum of weights by excluding weights on edges pointing to blocks that
393 // is either not in BlockFilter or is already in the current chain. Consider
394 // the following CFG:
387395 //
388396 // --->A
389397 // | / \
397405 // HotProb). If we exclude E that is not in BlockFilter when calculating the
398406 // probability of C->D, D will be selected and we will get A C D B as the
399407 // layout of this loop.
400 auto AdjustedSumProb = BranchProbability::getOne();
408 uint32_t AdjustedSumWeight = SumWeight;
401409 SmallVector Successors;
402410 for (MachineBasicBlock *Succ : BB->successors()) {
403411 bool SkipSucc = false;
415423 }
416424 }
417425 if (SkipSucc)
418 AdjustedSumProb -= MBPI->getEdgeProbability(BB, Succ);
426 AdjustedSumWeight -= MBPI->getEdgeWeight(BB, Succ) / WeightScale;
419427 else
420428 Successors.push_back(Succ);
421429 }
422430
423431 DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n");
424432 for (MachineBasicBlock *Succ : Successors) {
425 BranchProbability SuccProb;
426 uint32_t SuccProbN = MBPI->getEdgeProbability(BB, Succ).getNumerator();
427 uint32_t SuccProbD = AdjustedSumProb.getNumerator();
428 if (SuccProbN >= SuccProbD)
429 SuccProb = BranchProbability::getOne();
430 else
431 SuccProb = BranchProbability(SuccProbN, SuccProbD);
433 uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ);
434 BranchProbability SuccProb(SuccWeight / WeightScale, AdjustedSumWeight);
432435
433436 // If we outline optional branches, look whether Succ is unavoidable, i.e.
434437 // dominates all terminators of the MachineFunction. If it does, other
466469
467470 // Make sure that a hot successor doesn't have a globally more
468471 // important predecessor.
469 auto RealSuccProb = MBPI->getEdgeProbability(BB, Succ);
472 BranchProbability RealSuccProb(SuccWeight / WeightScale, SumWeight);
470473 BlockFrequency CandidateEdgeFreq =
471474 MBFI->getBlockFreq(BB) * RealSuccProb * HotProb.getCompl();
472475 bool BadCFGConflict = false;
492495 << " (prob)"
493496 << (SuccChain.LoopPredecessors != 0 ? " (CFG break)" : "")
494497 << "\n");
495 if (BestSucc && BestProb >= SuccProb)
498 if (BestSucc && BestWeight >= SuccWeight)
496499 continue;
497500 BestSucc = Succ;
498 BestProb = SuccProb;
501 BestWeight = SuccWeight;
499502 }
500503 return BestSucc;
501504 }
724727 MachineBasicBlock *OldExitingBB = ExitingBB;
725728 BlockFrequency OldBestExitEdgeFreq = BestExitEdgeFreq;
726729 bool HasLoopingSucc = false;
730 // FIXME: Due to the performance of the probability and weight routines in
731 // the MBPI analysis, we use the internal weights and manually compute the
732 // probabilities to avoid quadratic behavior.
733 uint32_t WeightScale = 0;
734 uint32_t SumWeight = MBPI->getSumForBlock(MBB, WeightScale);
727735 for (MachineBasicBlock *Succ : MBB->successors()) {
728736 if (Succ->isEHPad())
729737 continue;
737745 continue;
738746 }
739747
740 auto SuccProb = MBPI->getEdgeProbability(MBB, Succ);
748 uint32_t SuccWeight = MBPI->getEdgeWeight(MBB, Succ);
741749 if (LoopBlockSet.count(Succ)) {
742750 DEBUG(dbgs() << " looping: " << getBlockName(MBB) << " -> "
743 << getBlockName(Succ) << " (" << SuccProb << ")\n");
751 << getBlockName(Succ) << " (" << SuccWeight << ")\n");
744752 HasLoopingSucc = true;
745753 continue;
746754 }
752760 BlocksExitingToOuterLoop.insert(MBB);
753761 }
754762
763 BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight);
755764 BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(MBB) * SuccProb;
756765 DEBUG(dbgs() << " exiting: " << getBlockName(MBB) << " -> "
757766 << getBlockName(Succ) << " [L:" << SuccLoopDepth << "] (";
894903 // edge from the tail of the loop chain.
895904 SmallVector, 4> ExitsWithFreq;
896905 for (auto BB : LoopChain) {
897 auto LargestExitEdgeProb = BranchProbability::getZero();
906 uint32_t LargestExitEdgeWeight = 0;
898907 for (auto *Succ : BB->successors()) {
899908 BlockChain *SuccChain = BlockToChain[Succ];
900909 if (!LoopBlockSet.count(Succ) &&
901910 (!SuccChain || Succ == *SuccChain->begin())) {
902 auto SuccProb = MBPI->getEdgeProbability(BB, Succ);
903 LargestExitEdgeProb = std::max(LargestExitEdgeProb, SuccProb);
904 }
905 }
906 if (LargestExitEdgeProb > BranchProbability::getZero()) {
907 auto ExitFreq = MBFI->getBlockFreq(BB) * LargestExitEdgeProb;
911 uint32_t SuccWeight = MBPI->getEdgeWeight(BB, Succ);
912 LargestExitEdgeWeight = std::max(LargestExitEdgeWeight, SuccWeight);
913 }
914 }
915 if (LargestExitEdgeWeight > 0) {
916 uint32_t WeightScale = 0;
917 uint32_t SumWeight = MBPI->getSumForBlock(BB, WeightScale);
918 auto ExitFreq =
919 MBFI->getBlockFreq(BB) *
920 BranchProbability(LargestExitEdgeWeight / WeightScale, SumWeight);
908921 ExitsWithFreq.emplace_back(BB, ExitFreq);
909922 }
910923 }
12761289 }
12771290
12781291 // If PrevBB has a two-way branch, try to re-order the branches
1279 // such that we branch to the successor with higher probability first.
1292 // such that we branch to the successor with higher weight first.
12801293 if (TBB && !Cond.empty() && FBB &&
1281 MBPI->getEdgeProbability(PrevBB, FBB) >
1282 MBPI->getEdgeProbability(PrevBB, TBB) &&
1294 MBPI->getEdgeWeight(PrevBB, FBB) > MBPI->getEdgeWeight(PrevBB, TBB) &&
12831295 !TII->ReverseBranchCondition(Cond)) {
12841296 DEBUG(dbgs() << "Reverse order of the two branches: "
12851297 << getBlockName(PrevBB) << "\n");
1286 DEBUG(dbgs() << " Edge probability: "
1287 << MBPI->getEdgeProbability(PrevBB, FBB) << " vs "
1288 << MBPI->getEdgeProbability(PrevBB, TBB) << "\n");
1298 DEBUG(dbgs() << " Edge weight: " << MBPI->getEdgeWeight(PrevBB, FBB)
1299 << " vs " << MBPI->getEdgeWeight(PrevBB, TBB) << "\n");
12891300 DebugLoc dl; // FIXME: this is nowhere
12901301 TII->RemoveBranch(*PrevBB);
12911302 TII->InsertBranch(*PrevBB, FBB, TBB, Cond, dl);
2727
2828 void MachineBranchProbabilityInfo::anchor() { }
2929
30 uint32_t MachineBranchProbabilityInfo::getEdgeWeight(
31 const MachineBasicBlock *Src,
32 MachineBasicBlock::const_succ_iterator Dst) const {
33 return Src->getSuccProbability(Dst).getNumerator();
30 uint32_t MachineBranchProbabilityInfo::
31 getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const {
32 // First we compute the sum with 64-bits of precision, ensuring that cannot
33 // overflow by bounding the number of weights considered. Hopefully no one
34 // actually needs 2^32 successors.
35 assert(MBB->succ_size() < UINT32_MAX);
36 uint64_t Sum = 0;
37 Scale = 1;
38 for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
39 E = MBB->succ_end(); I != E; ++I) {
40 uint32_t Weight = getEdgeWeight(MBB, I);
41 Sum += Weight;
42 }
43
44 // If the computed sum fits in 32-bits, we're done.
45 if (Sum <= UINT32_MAX)
46 return Sum;
47
48 // Otherwise, compute the scale necessary to cause the weights to fit, and
49 // re-sum with that scale applied.
50 assert((Sum / UINT32_MAX) < UINT32_MAX);
51 Scale = (Sum / UINT32_MAX) + 1;
52 Sum = 0;
53 for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
54 E = MBB->succ_end(); I != E; ++I) {
55 uint32_t Weight = getEdgeWeight(MBB, I);
56 Sum += Weight / Scale;
57 }
58 assert(Sum <= UINT32_MAX);
59 return Sum;
3460 }
3561
36 uint32_t MachineBranchProbabilityInfo::getEdgeWeight(
37 const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const {
62 uint32_t MachineBranchProbabilityInfo::
63 getEdgeWeight(const MachineBasicBlock *Src,
64 MachineBasicBlock::const_succ_iterator Dst) const {
65 uint32_t Weight = Src->getSuccWeight(Dst);
66 if (!Weight)
67 return DEFAULT_WEIGHT;
68 return Weight;
69 }
70
71 uint32_t MachineBranchProbabilityInfo::
72 getEdgeWeight(const MachineBasicBlock *Src,
73 const MachineBasicBlock *Dst) const {
3874 // This is a linear search. Try to use the const_succ_iterator version when
3975 // possible.
4076 return getEdgeWeight(Src, std::find(Src->succ_begin(), Src->succ_end(), Dst));
41 }
42
43 BranchProbability MachineBranchProbabilityInfo::getEdgeProbability(
44 const MachineBasicBlock *Src,
45 MachineBasicBlock::const_succ_iterator Dst) const {
46 return Src->getSuccProbability(Dst);
47 }
48
49 BranchProbability MachineBranchProbabilityInfo::getEdgeProbability(
50 const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const {
51 // This is a linear search. Try to use the const_succ_iterator version when
52 // possible.
53 return getEdgeProbability(Src,
54 std::find(Src->succ_begin(), Src->succ_end(), Dst));
5577 }
5678
5779 bool
5880 MachineBranchProbabilityInfo::isEdgeHot(const MachineBasicBlock *Src,
5981 const MachineBasicBlock *Dst) const {
6082 // Hot probability is at least 4/5 = 80%
61 static BranchProbability HotProb(4, 5);
62 return getEdgeProbability(Src, Dst) > HotProb;
83 // FIXME: Compare against a static "hot" BranchProbability.
84 return getEdgeProbability(Src, Dst) > BranchProbability(4, 5);
6385 }
6486
6587 MachineBasicBlock *
6688 MachineBranchProbabilityInfo::getHotSucc(MachineBasicBlock *MBB) const {
67 auto MaxProb = BranchProbability::getZero();
89 uint32_t MaxWeight = 0;
6890 MachineBasicBlock *MaxSucc = nullptr;
6991 for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
7092 E = MBB->succ_end(); I != E; ++I) {
71 auto Prob = getEdgeProbability(MBB, I);
72 if (Prob > MaxProb) {
73 MaxProb = Prob;
93 uint32_t Weight = getEdgeWeight(MBB, I);
94 if (Weight > MaxWeight) {
95 MaxWeight = Weight;
7496 MaxSucc = *I;
7597 }
7698 }
7799
78 static BranchProbability HotProb(4, 5);
79 if (getEdgeProbability(MBB, MaxSucc) >= HotProb)
100 if (getEdgeProbability(MBB, MaxSucc) >= BranchProbability(4, 5))
80101 return MaxSucc;
81102
82103 return nullptr;
104 }
105
106 BranchProbability MachineBranchProbabilityInfo::getEdgeProbability(
107 const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const {
108 uint32_t Scale = 1;
109 uint32_t D = getSumForBlock(Src, Scale);
110 uint32_t N = getEdgeWeight(Src, Dst) / Scale;
111
112 return BranchProbability(N, D);
83113 }
84114
85115 raw_ostream &MachineBranchProbabilityInfo::printEdgeProbability(
744744 if (PredTBB)
745745 TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc());
746746
747 auto Prob = MBPI->getEdgeProbability(PredBB, TailBB);
747 uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB);
748748 PredBB->removeSuccessor(TailBB);
749749 unsigned NumSuccessors = PredBB->succ_size();
750750 assert(NumSuccessors <= 1);
751751 if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
752 PredBB->addSuccessor(NewTarget, Prob);
752 PredBB->addSuccessor(NewTarget, Weight);
753753
754754 TDBBs.push_back(PredBB);
755755 }
857857 "TailDuplicate called on block with multiple successors!");
858858 for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
859859 E = TailBB->succ_end(); I != E; ++I)
860 PredBB->addSuccessor(*I, MBPI->getEdgeProbability(TailBB, I));
860 PredBB->addSuccessor(*I, MBPI->getEdgeWeight(TailBB, I));
861861
862862 Changed = true;
863863 ++NumTailDups;
2121 const uint32_t BranchProbability::D;
2222
2323 raw_ostream &BranchProbability::print(raw_ostream &OS) const {
24 if (isUnknown())
25 return OS << "?%";
26
2724 // Get a percentage rounded to two decimal digits. This avoids
2825 // implementation-defined rounding inside printf.
2926 double Percent = rint(((double)N / D) * 100.0 * 100.0) / 100.0;
30 return OS << format("0x%08" PRIx32 " / 0x%08" PRIx32 " = %.2f%%", N, D,
31 Percent);
27 OS << format("0x%08" PRIx32 " / 0x%08" PRIx32 " = %.2f%%", N, D, Percent);
28 return OS;
3229 }
3330
3431 void BranchProbability::dump() const { print(dbgs()) << '\n'; }
4340 (Numerator * static_cast(D) + Denominator / 2) / Denominator;
4441 N = static_cast(Prob64);
4542 }
46 }
47
48 BranchProbability
49 BranchProbability::getBranchProbability(uint64_t Numerator,
50 uint64_t Denominator) {
51 assert(Numerator <= Denominator && "Probability cannot be bigger than 1!");
52 // Scale down Denominator to fit in a 32-bit integer.
53 int Scale = 0;
54 while (Denominator > UINT32_MAX) {
55 Denominator >>= 1;
56 Scale++;
57 }
58 return BranchProbability(Numerator >> Scale, Denominator);
5943 }
6044
6145 // If ConstD is not zero, then replace D by ConstD so that division and modulo
15691569
15701570 insertInstrBefore(DstBlk, AMDGPU::WHILELOOP, DebugLoc());
15711571 insertInstrEnd(DstBlk, AMDGPU::ENDLOOP, DebugLoc());
1572 DstBlk->replaceSuccessor(DstBlk, LandMBB);
1572 DstBlk->addSuccessor(LandMBB);
1573 DstBlk->removeSuccessor(DstBlk);
15731574 }
15741575
15751576
16641665 replaceInstrUseOfBlockWith(PredMBB, MBB, CloneMBB);
16651666 //srcBlk, oldBlk, newBlk
16661667
1667 PredMBB->replaceSuccessor(MBB, CloneMBB);
1668 PredMBB->removeSuccessor(MBB);
1669 PredMBB->addSuccessor(CloneMBB);
16681670
16691671 // add all successor to cloneBlk
16701672 cloneSuccessorList(CloneMBB, MBB);
22732273
22742274 // Update the CFG.
22752275 NewBB->addSuccessor(BB);
2276 JTBB->replaceSuccessor(BB, NewBB);
2276 JTBB->removeSuccessor(BB);
2277 JTBB->addSuccessor(NewBB);
22772278
22782279 ++NumJTInserted;
22792280 return NewBB;
73457345 }
73467346 }
73477347
7348 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
7348 BB->addSuccessor(DispatchBB);
73497349
73507350 // Find the invoke call and mark all of the callee-saved registers as
73517351 // 'implicit defined' so that they're spilled. This prevents code from
185185
186186 if (case1 || case2) {
187187 InvertAndChangeJumpTarget(MI, UncondTarget);
188 MBB->replaceSuccessor(JumpAroundTarget, UncondTarget);
188 MBB->removeSuccessor(JumpAroundTarget);
189 MBB->addSuccessor(UncondTarget);
189190
190191 // Remove the unconditional branch in LayoutSucc.
191192 LayoutSucc->erase(LayoutSucc->begin());
192 LayoutSucc->replaceSuccessor(UncondTarget, JumpAroundTarget);
193 LayoutSucc->removeSuccessor(UncondTarget);
194 LayoutSucc->addSuccessor(JumpAroundTarget);
193195
194196 // This code performs the conversion for case 2, which moves
195197 // the block to the fall-thru case (BB3 in the code above).
261261 static_cast(Subtarget.getInstrInfo());
262262
263263 MF->insert(FallThroughMBB, LongBrMBB);
264 MBB->replaceSuccessor(TgtMBB, LongBrMBB);
264 MBB->removeSuccessor(TgtMBB);
265 MBB->addSuccessor(LongBrMBB);
265266
266267 if (IsPIC) {
267268 MachineBasicBlock *BalTgtMBB = MF->CreateMachineBasicBlock(BB);
1313 br i1 undef, label %for.end, label %for.body
1414
1515 ; Before if conversion, we have
16 ; for.body -> lor.lhs.false.i (50%)
17 ; -> for.cond.backedge (50%)
18 ; lor.lhs.false.i -> for.cond.backedge (100%)
19 ; -> cond.false.i (0%)
16 ; for.body -> lor.lhs.false.i (62)
17 ; -> for.cond.backedge (62)
18 ; lor.lhs.false.i -> for.cond.backedge (1048575)
19 ; -> cond.false.i (1)
2020 ; Afer if conversion, we have
21 ; for.body -> for.cond.backedge (100%)
22 ; -> cond.false.i (0%)
21 ; for.body -> for.cond.backedge (130023362)
22 ; -> cond.false.i (62)
2323 ; CHECK: BB#1: derived from LLVM BB %for.body
24 ; CHECK: Successors according to CFG: BB#2(0x7ffffc00 / 0x80000000 = 100.00%) BB#4(0x00000400 / 0x80000000 = 0.00%)
24 ; CHECK: Successors according to CFG: BB#2(4294967291) BB#4(2048)
2525 for.body:
2626 br i1 undef, label %for.cond.backedge, label %lor.lhs.false.i, !prof !1
2727
1818 br i1 %9, label %return, label %bb2
1919
2020 ; CHECK: BB#2: derived from LLVM BB %bb2
21 ; CHECK: Successors according to CFG: BB#3({{[0-9a-fx/= ]+}}50.00%) BB#4({{[0-9a-fx/= ]+}}50.00%)
21 ; CHECK: Successors according to CFG: BB#3(4294967289) BB#4(4294967287)
2222
2323 bb2:
2424 %v10 = icmp eq i32 %3, 16
0 ; RUN: llc < %s -mtriple thumbv7s-apple-darwin -asm-verbose=false | FileCheck %s
1 ; RUN: llc < %s -mtriple thumbv7s-apple-darwin -asm-verbose=false -print-machineinstrs=if-converter 2>&1 | FileCheck --check-prefix=CHECK-PROB %s
1 ; RUN: llc < %s -mtriple thumbv7s-apple-darwin -asm-verbose=false -print-machineinstrs=if-converter 2>&1 | FileCheck --check-prefix=CHECK-WEIGHT %s
22
33 declare i32 @foo(i32)
44 declare i8* @bar(i32, i8*, i8*)
2828 ; CHECK-NEXT: [[FOOCALL]]:
2929 ; CHECK-NEXT: blx _foo
3030 ;
31 ; CHECK-PROB: BB#0:
32 ; CHECK-PROB: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}50.00%) BB#2({{[0-9a-fx/= ]+}}25.00%) BB#4({{[0-9a-fx/= ]+}}25.00%)
33 ; CHECK-PROB: BB#1:
34 ; CHECK-PROB: Successors according to CFG: BB#2({{[0-9a-fx/= ]+}}75.00%) BB#4({{[0-9a-fx/= ]+}}25.00%)
31 ; CHECK-WEIGHT: BB#0:
32 ; CHECK-WEIGHT: Successors according to CFG: BB#1(1073741824) BB#2(536870912) BB#4(536870912)
33 ; CHECK-WEIGHT: BB#1:
34 ; CHECK-WEIGHT: Successors according to CFG: BB#2(1610612736) BB#4(536870912)
3535
3636 define i32 @test(i32 %a, i32 %a2, i32* %p, i32* %p2) {
3737 entry:
88 ; = 0.2 * 0.4 + 0.8 * 0.7 = 0.64
99
1010 ; CHECK: # Machine code for function test0:
11 ; CHECK: Successors according to CFG: BB#{{[0-9]+}}({{[0-9a-fx/= ]+}}20.00%) BB#{{[0-9]+}}({{[0-9a-fx/= ]+}}80.00%)
11 ; CHECK: Successors according to CFG: BB#{{[0-9]+}}(13) BB#{{[0-9]+}}(24)
1212 ; CHECK: BB#{{[0-9]+}}:
1313 ; CHECK: BB#{{[0-9]+}}:
1414 ; CHECK: # End machine code for function test0.
22 ; RUN: | FileCheck %s
33
44 ; CHECK: Machine code for function test0:
5 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}3.12%) BB#2({{[0-9a-fx/= ]+}}96.88%)
5 ; CHECK: Successors according to CFG: BB#1(67108864) BB#2(2080374784)
66
77 define void @test0(i32 %a, i32 %b, i32* %c, i32* %d) {
88 entry:
2929 !0 = !{!"branch_weights", i32 4, i32 124}
3030
3131 ; CHECK: Machine code for function test1:
32 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}3.12%) BB#2({{[0-9a-fx/= ]+}}96.88%)
32 ; CHECK: Successors according to CFG: BB#1(67108864) BB#2(2080374784)
3333
3434 @g0 = common global i32 0, align 4
3535
1515 i64 5, label %sw.bb1
1616 ], !prof !0
1717 ; CHECK: BB#0: derived from LLVM BB %entry
18 ; CHECK: Successors according to CFG: BB#2({{[0-9a-fx/= ]+}}75.29%) BB#4({{[0-9a-fx/= ]+}}24.71%)
18 ; CHECK: Successors according to CFG: BB#2(1616928864) BB#4(530554784)
1919 ; CHECK: BB#4: derived from LLVM BB %entry
20 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}47.62%) BB#5({{[0-9a-fx/= ]+}}52.38%)
20 ; CHECK: Successors according to CFG: BB#1(252645135) BB#5(277909649)
2121 ; CHECK: BB#5: derived from LLVM BB %entry
22 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}36.36%) BB#3({{[0-9a-fx/= ]+}}63.64%)
22 ; CHECK: Successors according to CFG: BB#1(101058054) BB#3(176851595)
2323
2424 sw.bb:
2525 br label %return
6161 ; CHECK-LABEL: Machine code for function left_leaning_weight_balanced_tree:
6262 ; CHECK: BB#0: derived from LLVM BB %entry
6363 ; CHECK-NOT: Successors
64 ; CHECK: Successors according to CFG: BB#8({{[0-9a-fx/= ]+}}39.71%) BB#9({{[0-9a-fx/= ]+}}60.29%)
64 ; CHECK: Successors according to CFG: BB#8(852677332) BB#9(1294806318)
6565 }
6666
6767 !1 = !{!"branch_weights",
11 ; Check that the edge weights are updated correctly after if-conversion.
22
33 ; CHECK: BB#3:
4 ; CHECK: Successors according to CFG: BB#2({{[0-9a-fx/= ]+}}10.00%) BB#1({{[0-9a-fx/= ]+}}90.00%)
4 ; CHECK: Successors according to CFG: BB#2(214748365) BB#1(1932735283)
55 @a = external global i32
66 @d = external global i32
77
3434 # CHECK-LABEL: name: foo
3535 # CHECK: body: |
3636 # CHECK-NEXT: bb.0.entry:
37 # CHECK-NEXT: successors: %bb.1.less(0x40000000 / 0x80000000 = 50.00%), %bb.2.exit(0x40000000 / 0x80000000 = 50.00%)
37 # CHECK-NEXT: successors: %bb.1.less(0), %bb.2.exit(0)
3838 # CHECK-NEXT: liveins: %edi
3939 # CHECK: CMP32ri8 %edi, 10, implicit-def %eflags
4040 # CHECK-NEXT: JG_1 %bb.2.exit, implicit killed %eflags
7878 # CHECK-LABEL: name: bar
7979 # CHECK: body: |
8080 # CHECK-NEXT: bb.0.entry:
81 # CHECK-NEXT: successors: %bb.1.less(0x40000000 / 0x80000000 = 50.00%), %bb.2.exit(0x40000000 / 0x80000000 = 50.00%)
81 # CHECK-NEXT: successors: %bb.1.less(0), %bb.2.exit(0)
8282 # CHECK-NEXT: liveins: %edi
8383 # CHECK: CMP32ri8 %edi, 10, implicit-def %eflags
8484 # CHECK-NEXT: JG_1 %bb.2.exit, implicit killed %eflags
0 # RUN: llc -march=x86-64 -start-after branch-folder -stop-after branch-folder -o /dev/null %s | FileCheck %s
11 # This test ensures that the MIR parser parses basic block successors and
2 # probabilities correctly.
2 # weights correctly.
33
44 --- |
55
2020 name: foo
2121 body: |
2222 ; CHECK-LABEL: bb.0.entry:
23 ; CHECK: successors: %bb.1.less({{[0-9a-fx/= ]+}}33.00%), %bb.2.exit({{[0-9a-fx/= ]+}}67.00%)
23 ; CHECK: successors: %bb.1.less(16), %bb.2.exit(32)
2424 ; CHECK-LABEL: bb.1.less:
2525 bb.0.entry:
26 successors: %bb.1.less (33), %bb.2.exit(67)
26 successors: %bb.1.less (16), %bb.2.exit(32)
2727 liveins: %edi
2828
2929 CMP32ri8 %edi, 10, implicit-def %eflags
3131 name: foo
3232 body: |
3333 ; CHECK-LABEL: bb.0.entry:
34 ; CHECK: successors: %bb.1.less(0x40000000 / 0x80000000 = 50.00%), %bb.2.exit(0x40000000 / 0x80000000 = 50.00%)
34 ; CHECK: successors: %bb.1.less(0), %bb.2.exit(0)
3535 ; CHECK-LABEL: bb.1.less:
3636 bb.0.entry:
3737 successors: %bb.1.less, %bb.2.exit
5757 ; Verify that we can have multiple lists of successors that will be merged
5858 ; into one.
5959 ; CHECK-LABEL: bb.0.entry:
60 ; CHECK: successors: %bb.1(0x80000000 / 0x80000000 = 100.00%), %bb.2(0x00000000 / 0x80000000 = 0.00%)
60 ; CHECK: successors: %bb.1(0), %bb.2(0)
6161 bb.0.entry:
6262 liveins: %edi
6363 successors: %bb.1
1717 %or.cond = or i1 %tobool, %cmp4
1818 br i1 %or.cond, label %for.inc20, label %for.inc, !prof !0
1919 ; CHECK: BB#1: derived from LLVM BB %for.cond2
20 ; CHECK: Successors according to CFG: BB#3({{[0-9a-fx/= ]+}}1.53%) BB#4({{[0-9a-fx/= ]+}}98.47%)
20 ; CHECK: Successors according to CFG: BB#3(32756933) BB#4(2114726715)
2121 ; CHECK: BB#4: derived from LLVM BB %for.cond2
22 ; CHECK: Successors according to CFG: BB#3({{[0-9a-fx/= ]+}}1.55%) BB#2({{[0-9a-fx/= ]+}}98.45%)
22 ; CHECK: Successors according to CFG: BB#3(33264335) BB#2(2114219313)
2323
2424 for.inc: ; preds = %for.cond2
2525 %shl = shl i32 %bit.0, 1
11
22 ; Check if the edge weight to the catchpad is calculated correctly.
33
4 ; CHECK: Successors according to CFG: BB#3(0x7ffff100 / 0x80000000 = 100.00%) BB#1(0x00000800 / 0x80000000 = 0.00%) BB#4(0x00000400 / 0x80000000 = 0.00%) BB#6(0x00000200 / 0x80000000 = 0.00%) BB#8(0x00000100 / 0x80000000 = 0.00%)
4 ; CHECK: Successors according to CFG: BB#3(2147481600) BB#1(2048) BB#4(1024) BB#6(512) BB#8(256)
55
66 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
77 target triple = "x86_64--windows-msvc18.0.0"
11 ; RUN: llc -mtriple=x86_64-apple-darwin -print-machineinstrs=expand-isel-pseudos -enable-selectiondag-sp=false %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=IR
22
33 ; SELDAG: # Machine code for function test_branch_weights:
4 ; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]]({{[0-9a-fx/= ]+}}100.00%) BB#[[FAILURE:[0-9]+]]
4 ; SELDAG: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](2147481600) BB#[[FAILURE:[0-9]+]](2048)
55 ; SELDAG: BB#[[FAILURE]]:
66 ; SELDAG: CALL64pcrel32
77 ; SELDAG: BB#[[SUCCESS]]:
88
99 ; IR: # Machine code for function test_branch_weights:
10 ; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]]({{[0-9a-fx/= ]+}}100.00%) BB#[[FAILURE:[0-9]+]]
10 ; IR: Successors according to CFG: BB#[[SUCCESS:[0-9]+]](2147481600) BB#[[FAILURE:[0-9]+]](2048)
1111 ; IR: BB#[[SUCCESS]]:
1212 ; IR: BB#[[FAILURE]]:
1313 ; IR: CALL64pcrel32
3333 ; CHECK: BB#0:
3434 ; BB#0 to BB#4: [0, 1133] (65 = 60 + 5)
3535 ; BB#0 to BB#5: [1134, UINT32_MAX] (25 = 20 + 5)
36 ; CHECK: Successors according to CFG: BB#4({{[0-9a-fx/= ]+}}72.22%) BB#5({{[0-9a-fx/= ]+}}27.78%)
36 ; CHECK: Successors according to CFG: BB#4(1550960411) BB#5(596523235)
3737 ;
3838 ; CHECK: BB#4:
3939 ; BB#4 to BB#1: [155, 159] (50)
4040 ; BB#4 to BB#5: [0, 1133] - [155, 159] (15 = 10 + 5)
41 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}76.92%) BB#7({{[0-9a-fx/= ]+}}23.08%)
41 ; CHECK: Successors according to CFG: BB#1(1193046470) BB#7(357913941)
4242 ;
4343 ; CHECK: BB#5:
4444 ; BB#5 to BB#1: {1140} (10)
4545 ; BB#5 to BB#6: [1134, UINT32_MAX] - {1140} (15 = 10 + 5)
46 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}40.00%) BB#6({{[0-9a-fx/= ]+}}60.00%)
46 ; CHECK: Successors according to CFG: BB#1(238609294) BB#6(357913941)
4747 ;
4848 ; CHECK: BB#6:
4949 ; BB#6 to BB#1: {1134} (10)
5050 ; BB#6 to BB#2: [1134, UINT32_MAX] - {1134, 1140} (5)
51 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}66.67%) BB#2({{[0-9a-fx/= ]+}}33.33%)
51 ; CHECK: Successors according to CFG: BB#1(238609294) BB#2(119304647)
5252 }
5353
5454 ; CHECK-LABEL: test2
101101 ; CHECK: BB#0:
102102 ; BB#0 to BB#6: {0} + [15, UINT32_MAX] (5)
103103 ; BB#0 to BB#8: [1, 14] (jump table) (65 = 60 + 5)
104 ; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}7.14%) BB#8({{[0-9a-fx/= ]+}}92.86%
104 ; CHECK: Successors according to CFG: BB#6(153391689) BB#8(1994091957)
105105 ;
106106 ; CHECK: BB#8:
107107 ; BB#8 to BB#1: {1} (10)
110110 ; BB#8 to BB#3: {11} (10)
111111 ; BB#8 to BB#4: {12} (10)
112112 ; BB#8 to BB#5: {13, 14} (20)
113 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}14.29%) BB#6({{[0-9a-fx/= ]+}}7.14%) BB#2({{[0-9a-fx/= ]+}}14.29%) BB#3({{[0-9a-fx/= ]+}}14.29%) BB#4({{[0-9a-fx/= ]+}}14.29%) BB#5({{[0-9a-fx/= ]+}}28.57%)
113 ; CHECK: Successors according to CFG: BB#1(306783378) BB#6(153391689) BB#2(306783378) BB#3(306783378) BB#4(306783378) BB#5(613566756)
114114 }
115115
116116 ; CHECK-LABEL: test3
162162 ; CHECK: BB#0:
163163 ; BB#0 to BB#6: [0, 9] + [15, UINT32_MAX] {10}
164164 ; BB#0 to BB#8: [10, 14] (jump table) (50)
165 ; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}16.67%) BB#8({{[0-9a-fx/= ]+}}83.33%)
165 ; CHECK: Successors according to CFG: BB#6(357913941) BB#8(1789569705)
166166 ;
167167 ; CHECK: BB#8:
168168 ; BB#8 to BB#1: {10} (10)
170170 ; BB#8 to BB#3: {12} (10)
171171 ; BB#8 to BB#4: {13} (10)
172172 ; BB#8 to BB#5: {14} (10)
173 ; CHECK: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}20.00%) BB#2({{[0-9a-fx/= ]+}}20.00%) BB#3({{[0-9a-fx/= ]+}}20.00%) BB#4({{[0-9a-fx/= ]+}}20.00%) BB#5({{[0-9a-fx/= ]+}}20.00%)
173 ; CHECK: Successors according to CFG: BB#1(357913941) BB#2(357913941) BB#3(357913941) BB#4(357913941) BB#5(357913941)
174174 }
175175
176176 ; CHECK-LABEL: test4
215215 ; CHECK: BB#0:
216216 ; BB#0 to BB#6: [0, 110] + [116, UINT32_MAX] (20)
217217 ; BB#0 to BB#7: [111, 115] (bit test) (50)
218 ; CHECK: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}28.57%) BB#7({{[0-9a-fx/= ]+}}71.43%)
218 ; CHECK: Successors according to CFG: BB#6(613566756) BB#7(1533916890)
219219 ;
220220 ; CHECK: BB#7:
221221 ; BB#7 to BB#2: {111, 114, 115} (30)
222222 ; BB#7 to BB#3: {112, 113} (20)
223 ; CHECK: Successors according to CFG: BB#2({{[0-9a-fx/= ]+}}60.00%) BB#3({{[0-9a-fx/= ]+}}40.00%)
223 ; CHECK: Successors according to CFG: BB#2(920350134) BB#3(613566756)
224224 }
225225
226226 ; CHECK-LABEL: test5
272272 ; CHECK: BB#0:
273273 ; BB#0 to BB#6: [10, UINT32_MAX] (15)
274274 ; BB#0 to BB#8: [1, 5, 7, 9] (jump table) (45)
275 ; CHECK: Successors according to CFG: BB#8({{[0-9a-fx/= ]+}}25.00%) BB#9({{[0-9a-fx/= ]+}}75.00%)
275 ; CHECK: Successors according to CFG: BB#8(536870912) BB#9(1610612734)
276276 }
277277
278278 !1 = !{!"branch_weights", i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10}
0 ; RUN: llc -mtriple=i686-pc-gnu-linux < %s | FileCheck %s -check-prefix=CHECK
1 ; RUN: llc -mtriple=i686-pc-gnu-linux -print-machineinstrs=expand-isel-pseudos %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=CHECK-JT-PROB
1 ; RUN: llc -mtriple=i686-pc-gnu-linux -print-machineinstrs=expand-isel-pseudos %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=CHECK-JT-WEIGHT
22
33
44 ; An unreachable default destination is replaced with the most popular case label.
5353 ; Check if branch probabilities are correctly assigned to the jump table.
5454
5555 define void @bar(i32 %x, i32* %to) {
56 ; CHECK-JT-PROB-LABEL: bar:
57 ; CHECK-JT-PROB: Successors according to CFG: BB#6({{[0-9a-fx/= ]+}}14.29%) BB#8({{[0-9a-fx/= ]+}}85.71%)
58 ; CHECK-JT-PROB: Successors according to CFG: BB#1({{[0-9a-fx/= ]+}}16.67%) BB#2({{[0-9a-fx/= ]+}}16.67%) BB#3({{[0-9a-fx/= ]+}}16.67%) BB#4({{[0-9a-fx/= ]+}}16.67%) BB#5({{[0-9a-fx/= ]+}}33.33%)
56 ; CHECK-JT-WEIGHT-LABEL: bar:
57 ; CHECK-JT-WEIGHT: Successors according to CFG: BB#6(306783378) BB#8(1840700268)
58 ; CHECK-JT-WEIGHT: Successors according to CFG: BB#1(306783378) BB#2(306783378) BB#3(306783378) BB#4(306783378) BB#5(613566756)
5959
6060 entry:
6161 switch i32 %x, label %default [