llvm.org GIT mirror llvm / 5ae2b85
Normalize MBB's successors' probabilities in several locations. This patch adds some missing calls to MBB::normalizeSuccProbs() in several locations where it should be called. Those places are found by checking if the sum of successors' probabilities is approximate one in MachineBlockPlacement pass with some instrumented code (not in this patch). Differential revision: http://reviews.llvm.org/D15259 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@255455 91177308-0d34-0410-b5e6-96231b3b80d8 Cong Hou 4 years ago
12 changed file(s) with 68 addition(s) and 27 deletion(s). Raw diff Collapse all Expand all
453453 void setSuccProbability(succ_iterator I, BranchProbability Prob);
454454
455455 /// Normalize probabilities of all successors so that the sum of them becomes
456 /// one.
456 /// one. This is usually done when the current update on this MBB is done, and
457 /// the sum of its successors' probabilities is not guaranteed to be one. The
458 /// user is responsible for the correct use of this function.
459 /// MBB::removeSuccessor() has an option to do this automatically.
457460 void normalizeSuccProbs() {
458461 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
459462 }
460463
464 /// Validate successors' probabilities and check if the sum of them is
465 /// approximate one. This only works in DEBUG mode.
466 void validateSuccProbs() const;
467
461468 /// Remove successor from the successors list of this MachineBasicBlock. The
462469 /// Predecessors list of Succ is automatically updated.
463 void removeSuccessor(MachineBasicBlock *Succ);
470 /// If NormalizeSuccProbs is true, then normalize successors' probabilities
471 /// after the successor is removed.
472 void removeSuccessor(MachineBasicBlock *Succ,
473 bool NormalizeSuccProbs = false);
464474
465475 /// Remove specified successor from the successors list of this
466476 /// MachineBasicBlock. The Predecessors list of Succ is automatically updated.
477 /// If NormalizeSuccProbs is true, then normalize successors' probabilities
478 /// after the successor is removed.
467479 /// Return the iterator to the element after the one removed.
468 succ_iterator removeSuccessor(succ_iterator I);
480 succ_iterator removeSuccessor(succ_iterator I,
481 bool NormalizeSuccProbs = false);
469482
470483 /// Replace successor OLD with NEW and update probability info.
471484 void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New);
537537
538538 // Fix up the CFG, temporarily leave Head without any successors.
539539 Head->removeSuccessor(TBB);
540 Head->removeSuccessor(FBB);
540 Head->removeSuccessor(FBB, true);
541541 if (TBB != Tail)
542 TBB->removeSuccessor(Tail);
542 TBB->removeSuccessor(Tail, true);
543543 if (FBB != Tail)
544 FBB->removeSuccessor(Tail);
544 FBB->removeSuccessor(Tail, true);
545545
546546 // Fix up Head's terminators.
547547 // It should become a single branch or a fallthrough.
11121112
11131113 // RemoveExtraEdges won't work if the block has an unanalyzable branch, so
11141114 // explicitly remove CvtBBI as a successor.
1115 BBI.BB->removeSuccessor(CvtBBI->BB);
1115 BBI.BB->removeSuccessor(CvtBBI->BB, true);
11161116 } else {
11171117 RemoveKills(CvtBBI->BB->begin(), CvtBBI->BB->end(), DontKill, *TRI);
11181118 PredicateBlock(*CvtBBI, CvtBBI->BB->end(), Cond);
12251225
12261226 // RemoveExtraEdges won't work if the block has an unanalyzable branch, so
12271227 // explicitly remove CvtBBI as a successor.
1228 BBI.BB->removeSuccessor(CvtBBI->BB);
1228 BBI.BB->removeSuccessor(CvtBBI->BB, true);
12291229 } else {
12301230 // Predicate the 'true' block after removing its branch.
12311231 CvtBBI->NonPredSize -= TII->RemoveBranch(*CvtBBI->BB);
15111511 // which can happen here if TailBB is unanalyzable and is merged, so
15121512 // explicitly remove BBI1 and BBI2 as successors.
15131513 BBI.BB->removeSuccessor(BBI1->BB);
1514 BBI.BB->removeSuccessor(BBI2->BB);
1514 BBI.BB->removeSuccessor(BBI2->BB, true);
15151515 RemoveExtraEdges(BBI);
15161516
15171517 // Update block info.
17051705
17061706 if (AddEdges) {
17071707 // If the edge from ToBBI.BB to Succ already exists, update the
1708 // probability of this edge by adding NewWeight to it. An example is shown
1708 // probability of this edge by adding NewProb to it. An example is shown
17091709 // below, in which A is ToBBI.BB and B is FromBBI.BB. In this case we
17101710 // don't have to set C as A's successor as it already is. We only need to
17111711 // update the edge probability on A->C. Note that B will not be
17391739 if (NBB && !FromBBI.BB->isSuccessor(NBB))
17401740 FromBBI.BB->addSuccessor(NBB);
17411741
1742 // Normalize the probabilities of ToBBI.BB's successors with all adjustment
1743 // we've done above.
1744 ToBBI.BB->normalizeSuccProbs();
1745
17421746 ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end());
17431747 FromBBI.Predicate.clear();
17441748
505505 }
506506 }
507507
508 void MachineBasicBlock::validateSuccProbs() const {
509 #ifndef NDEBUG
510 int64_t Sum = 0;
511 for (auto Prob : Probs)
512 Sum += Prob.getNumerator();
513 // Due to precision issue, we assume that the sum of probabilities is one if
514 // the difference between the sum of their numerators and the denominator is
515 // no greater than the number of successors.
516 assert(std::abs(Sum - BranchProbability::getDenominator()) <=
517 Probs.size() &&
518 "The sum of successors's probabilities exceeds one.");
519 #endif // NDEBUG
520 }
521
508522 void MachineBasicBlock::addSuccessor(MachineBasicBlock *Succ,
509523 BranchProbability Prob) {
510524 // Probability list is either empty (if successor list isn't empty, this means
524538 Succ->addPredecessor(this);
525539 }
526540
527 void MachineBasicBlock::removeSuccessor(MachineBasicBlock *Succ) {
541 void MachineBasicBlock::removeSuccessor(MachineBasicBlock *Succ,
542 bool NormalizeSuccProbs) {
528543 succ_iterator I = std::find(Successors.begin(), Successors.end(), Succ);
529 removeSuccessor(I);
544 removeSuccessor(I, NormalizeSuccProbs);
530545 }
531546
532547 MachineBasicBlock::succ_iterator
533 MachineBasicBlock::removeSuccessor(succ_iterator I) {
548 MachineBasicBlock::removeSuccessor(succ_iterator I, bool NormalizeSuccProbs) {
534549 assert(I != Successors.end() && "Not a current successor!");
535550
536551 // If probability list is empty it means we don't use it (disabled
538553 if (!Probs.empty()) {
539554 probability_iterator WI = getProbabilityIterator(I);
540555 Probs.erase(WI);
556 if (NormalizeSuccProbs)
557 normalizeSuccProbs();
541558 }
542559
543560 (*I)->removePredecessor(this);
635652 MO.setMBB(this);
636653 }
637654 }
655 normalizeSuccProbs();
638656 }
639657
640658 bool MachineBasicBlock::isPredecessor(const MachineBasicBlock *MBB) const {
10871105 }
10881106 }
10891107
1108 if (Changed)
1109 normalizeSuccProbs();
10901110 return Changed;
10911111 }
10921112
22692269 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
22702270 addSuccessorWithProb(IndirectBrMBB, Succ);
22712271 }
2272 IndirectBrMBB->normalizeSuccProbs();
22722273
22732274 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
22742275 MVT::Other, getControlRoot(),
750750 assert(NumSuccessors <= 1);
751751 if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
752752 PredBB->addSuccessor(NewTarget, Prob);
753 PredBB->normalizeSuccProbs();
753754
754755 TDBBs.push_back(PredBB);
755756 }
566566 // All CmpBB instructions are moved into Head, and CmpBB is deleted.
567567 // Update the CFG first.
568568 updateTailPHIs();
569 Head->removeSuccessor(CmpBB);
570 CmpBB->removeSuccessor(Tail);
569 Head->removeSuccessor(CmpBB, true);
570 CmpBB->removeSuccessor(Tail, true);
571571 Head->transferSuccessorsAndUpdatePHIs(CmpBB);
572572 DebugLoc TermDL = Head->getFirstTerminator()->getDebugLoc();
573573 TII->RemoveBranch(*Head);
11631163
11641164 for (SmallVectorImpl::iterator It = ContMBB.begin(),
11651165 E = ContMBB.end(); It != E; ++It) {
1166 (*It)->removeSuccessor(LoopHeader);
1166 (*It)->removeSuccessor(LoopHeader, true);
11671167 }
11681168
11691169 numLoopcontPatternMatch += NumCont;
14861486 );
14871487 DstMBB->splice(DstMBB->end(), SrcMBB, SrcMBB->begin(), SrcMBB->end());
14881488
1489 DstMBB->removeSuccessor(SrcMBB);
1489 DstMBB->removeSuccessor(SrcMBB, true);
14901490 cloneSuccessorList(DstMBB, SrcMBB);
14911491
14921492 removeSuccessor(SrcMBB);
15361536
15371537 if (TrueMBB) {
15381538 MBB->splice(I, TrueMBB, TrueMBB->begin(), TrueMBB->end());
1539 MBB->removeSuccessor(TrueMBB);
1539 MBB->removeSuccessor(TrueMBB, true);
15401540 if (LandMBB && TrueMBB->succ_size()!=0)
1541 TrueMBB->removeSuccessor(LandMBB);
1541 TrueMBB->removeSuccessor(LandMBB, true);
15421542 retireBlock(TrueMBB);
15431543 MLI->removeBlock(TrueMBB);
15441544 }
15471547 insertInstrBefore(I, AMDGPU::ELSE);
15481548 MBB->splice(I, FalseMBB, FalseMBB->begin(),
15491549 FalseMBB->end());
1550 MBB->removeSuccessor(FalseMBB);
1550 MBB->removeSuccessor(FalseMBB, true);
15511551 if (LandMBB && FalseMBB->succ_size() != 0)
1552 FalseMBB->removeSuccessor(LandMBB);
1552 FalseMBB->removeSuccessor(LandMBB, true);
15531553 retireBlock(FalseMBB);
15541554 MLI->removeBlock(FalseMBB);
15551555 }
15901590 //now branchInst can be erase safely
15911591 BranchMI->eraseFromParent();
15921592 //now take care of successors, retire blocks
1593 ExitingMBB->removeSuccessor(LandMBB);
1593 ExitingMBB->removeSuccessor(LandMBB, true);
15941594 }
15951595
15961596 void AMDGPUCFGStructurizer::settleLoopcontBlock(MachineBasicBlock *ContingMBB,
17561756 DEBUG(dbgs() << "Removing unneeded cond branch instr: " << *BranchMI);
17571757 BranchMI->eraseFromParent();
17581758 SHOWNEWBLK(MBB1, "Removing redundant successor");
1759 MBB->removeSuccessor(MBB1);
1759 MBB->removeSuccessor(MBB1, true);
17601760 }
17611761
17621762 void AMDGPUCFGStructurizer::addDummyExitBlock(
73967396 }
73977397
73987398 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
7399 BB->normalizeSuccProbs();
73997400
74007401 // Find the invoke call and mark all of the callee-saved registers as
74017402 // 'implicit defined' so that they're spilled. This prevents code from
938938 B->removeSuccessor(B->succ_begin());
939939
940940 for (auto I = B->pred_begin(), E = B->pred_end(); I != E; ++I)
941 (*I)->removeSuccessor(B);
941 (*I)->removeSuccessor(B, true);
942942
943943 Deleted.insert(B);
944944 MDT->eraseNode(B);
10001000 MachineBasicBlock::succ_iterator I, E = SuccB->succ_end();
10011001 for (I = SuccB->succ_begin(); I != E; ++I)
10021002 PredB->addSuccessor(*I);
1003 PredB->normalizeSuccProbs();
10031004 replacePhiEdges(SuccB, PredB);
10041005 removeBlock(SuccB);
10051006 if (!TermOk)
147147 // Insert NewMBB and fix control flow.
148148 MachineBasicBlock *Tgt = getTargetMBB(*FirstBr);
149149 NewMBB->transferSuccessors(MBB);
150 NewMBB->removeSuccessor(Tgt);
150 NewMBB->removeSuccessor(Tgt, true);
151151 MBB->addSuccessor(NewMBB);
152152 MBB->addSuccessor(Tgt);
153153 MF->insert(std::next(MachineFunction::iterator(MBB)), NewMBB);
152152 }
153153
154154 for (unsigned i = 0, ie = PredToRemove.size(); i != ie; ++i)
155 PredToRemove[i]->removeSuccessor(&ReturnMBB);
155 PredToRemove[i]->removeSuccessor(&ReturnMBB, true);
156156
157157 if (Changed && !ReturnMBB.hasAddressTaken()) {
158158 // We now might be able to merge this blr-only block into its
162162 if (PrevMBB.isLayoutSuccessor(&ReturnMBB) && PrevMBB.canFallThrough()) {
163163 // Move the blr into the preceding block.
164164 PrevMBB.splice(PrevMBB.end(), &ReturnMBB, I);
165 PrevMBB.removeSuccessor(&ReturnMBB);
165 PrevMBB.removeSuccessor(&ReturnMBB, true);
166166 }
167167 }
168168