llvm.org GIT mirror llvm / ae7ddf0
[LICM] Make LICM able to hoist phis The general approach taken is to make note of loop invariant branches, then when we see something conditional on that branch, such as a phi, we create a copy of the branch and (empty versions of) its successors and hoist using that. This has no impact by itself that I've been able to see, as LICM typically doesn't see such phis as they will have been converted into selects by the time LICM is run, but once we start doing phi-to-select conversion later it will be important. Differential Revision: https://reviews.llvm.org/D52827 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@347190 91177308-0d34-0410-b5e6-96231b3b80d8 John Brawn 10 months ago
3 changed file(s) with 1480 addition(s) and 27 deletion(s). Raw diff Collapse all Expand all
3030 //===----------------------------------------------------------------------===//
3131
3232 #include "llvm/Transforms/Scalar/LICM.h"
33 #include "llvm/ADT/SetOperations.h"
3334 #include "llvm/ADT/Statistic.h"
3435 #include "llvm/Analysis/AliasAnalysis.h"
3536 #include "llvm/Analysis/AliasSetTracker.h"
4041 #include "llvm/Analysis/GuardUtils.h"
4142 #include "llvm/Analysis/Loads.h"
4243 #include "llvm/Analysis/LoopInfo.h"
44 #include "llvm/Analysis/LoopIterator.h"
4345 #include "llvm/Analysis/LoopPass.h"
4446 #include "llvm/Analysis/MemoryBuiltins.h"
4547 #include "llvm/Analysis/MemorySSA.h"
7476
7577 #define DEBUG_TYPE "licm"
7678
79 STATISTIC(NumCreatedBlocks, "Number of blocks created");
80 STATISTIC(NumClonedBranches, "Number of branches cloned");
7781 STATISTIC(NumSunk, "Number of instructions sunk out of loop");
7882 STATISTIC(NumHoisted, "Number of instructions hoisted out of loop");
7983 STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk");
102106 const LoopSafetyInfo *SafetyInfo,
103107 TargetTransformInfo *TTI, bool &FreeInLoop);
104108 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
105 ICFLoopSafetyInfo *SafetyInfo,
109 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
106110 OptimizationRemarkEmitter *ORE);
107111 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
108112 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
436440 return Changed;
437441 }
438442
443 // This is a helper class for hoistRegion to make it able to hoist control flow
444 // in order to be able to hoist phis. The way this works is that we initially
445 // start hoisting to the loop preheader, and when we see a loop invariant branch
446 // we make note of this. When we then come to hoist an instruction that's
447 // conditional on such a branch we duplicate the branch and the relevant control
448 // flow, then hoist the instruction into the block corresponding to its original
449 // block in the duplicated control flow.
450 class ControlFlowHoister {
451 private:
452 // Information about the loop we are hoisting from
453 LoopInfo *LI;
454 DominatorTree *DT;
455 Loop *CurLoop;
456
457 // A map of blocks in the loop to the block their instructions will be hoisted
458 // to.
459 DenseMap HoistDestinationMap;
460
461 // The branches that we can hoist, mapped to the block that marks a
462 // convergence point of their control flow.
463 DenseMap HoistableBranches;
464
465 public:
466 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop)
467 : LI(LI), DT(DT), CurLoop(CurLoop) {}
468
469 void registerPossiblyHoistableBranch(BranchInst *BI) {
470 // We can only hoist conditional branches with loop invariant operands.
471 if (!BI->isConditional() || !CurLoop->hasLoopInvariantOperands(BI))
472 return;
473
474 // The branch destinations need to be in the loop, and we don't gain
475 // anything by duplicating conditional branches with duplicate successors,
476 // as it's essentially the same as an unconditional branch.
477 BasicBlock *TrueDest = BI->getSuccessor(0);
478 BasicBlock *FalseDest = BI->getSuccessor(1);
479 if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
480 TrueDest == FalseDest)
481 return;
482
483 // We can hoist BI if one branch destination is the successor of the other,
484 // or both have common successor which we check by seeing if the
485 // intersection of their successors is non-empty.
486 // TODO: This could be expanded to allowing branches where both ends
487 // eventually converge to a single block.
488 SmallPtrSet TrueDestSucc, FalseDestSucc;
489 TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
490 FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
491 BasicBlock *CommonSucc = nullptr;
492 if (TrueDestSucc.count(FalseDest)) {
493 CommonSucc = FalseDest;
494 } else if (FalseDestSucc.count(TrueDest)) {
495 CommonSucc = TrueDest;
496 } else {
497 set_intersect(TrueDestSucc, FalseDestSucc);
498 // If there's one common successor use that.
499 if (TrueDestSucc.size() == 1)
500 CommonSucc = *TrueDestSucc.begin();
501 // If there's more than one pick whichever appears first in the block list
502 // (we can't use the value returned by TrueDestSucc.begin() as it's
503 // unpredicatable which element gets returned).
504 else if (!TrueDestSucc.empty()) {
505 Function *F = TrueDest->getParent();
506 auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
507 auto It = std::find_if(F->begin(), F->end(), IsSucc);
508 assert(It != F->end() && "Could not find successor in function");
509 CommonSucc = &*It;
510 }
511 }
512 // The common successor has to be dominated by the branch, as otherwise
513 // there will be some other path to the successor that will not be
514 // controlled by this branch so any phi we hoist would be controlled by the
515 // wrong condition. This also takes care of avoiding hoisting of loop back
516 // edges.
517 // TODO: In some cases this could be relaxed if the successor is dominated
518 // by another block that's been hoisted and we can guarantee that the
519 // control flow has been replicated exactly.
520 if (CommonSucc && DT->dominates(BI, CommonSucc))
521 HoistableBranches[BI] = CommonSucc;
522 }
523
524 bool canHoistPHI(PHINode *PN) {
525 // The phi must have loop invariant operands.
526 if (!CurLoop->hasLoopInvariantOperands(PN))
527 return false;
528 // We can hoist phis if the block they are in is the target of hoistable
529 // branches which cover all of the predecessors of the block.
530 SmallPtrSet PredecessorBlocks;
531 BasicBlock *BB = PN->getParent();
532 for (BasicBlock *PredBB : predecessors(BB))
533 PredecessorBlocks.insert(PredBB);
534 // If we have less predecessor blocks than predecessors then the phi will
535 // have more than one incoming value for the same block which we can't
536 // handle.
537 // TODO: This could be handled be erasing some of the duplicate incoming
538 // values.
539 if (PredecessorBlocks.size() != pred_size(BB))
540 return false;
541 for (auto &Pair : HoistableBranches) {
542 if (Pair.second == BB) {
543 // Which blocks are predecessors via this branch depends on if the
544 // branch is triangle-like or diamond-like.
545 if (Pair.first->getSuccessor(0) == BB) {
546 PredecessorBlocks.erase(Pair.first->getParent());
547 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
548 } else if (Pair.first->getSuccessor(1) == BB) {
549 PredecessorBlocks.erase(Pair.first->getParent());
550 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
551 } else {
552 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
553 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
554 }
555 }
556 }
557 // PredecessorBlocks will now be empty if for every predecessor of BB we
558 // found a hoistable branch source.
559 return PredecessorBlocks.empty();
560 }
561
562 BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
563 // If BB has already been hoisted, return that
564 if (HoistDestinationMap.count(BB))
565 return HoistDestinationMap[BB];
566
567 // Check if this block is conditional based on a pending branch
568 auto HasBBAsSuccessor =
569 [&](DenseMap::value_type &Pair) {
570 return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
571 Pair.first->getSuccessor(1) == BB);
572 };
573 auto It = std::find_if(HoistableBranches.begin(), HoistableBranches.end(),
574 HasBBAsSuccessor);
575
576 // If not involved in a pending branch, hoist to preheader
577 BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
578 if (It == HoistableBranches.end()) {
579 LLVM_DEBUG(dbgs() << "LICM using " << InitialPreheader->getName()
580 << " as hoist destination for " << BB->getName()
581 << "\n");
582 HoistDestinationMap[BB] = InitialPreheader;
583 return InitialPreheader;
584 }
585 BranchInst *BI = It->first;
586 assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==
587 HoistableBranches.end() &&
588 "BB is expected to be the target of at most one branch");
589
590 LLVMContext &C = BB->getContext();
591 BasicBlock *TrueDest = BI->getSuccessor(0);
592 BasicBlock *FalseDest = BI->getSuccessor(1);
593 BasicBlock *CommonSucc = HoistableBranches[BI];
594 BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());
595
596 // Create hoisted versions of blocks that currently don't have them
597 auto CreateHoistedBlock = [&](BasicBlock *Orig) {
598 if (HoistDestinationMap.count(Orig))
599 return HoistDestinationMap[Orig];
600 BasicBlock *New =
601 BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
602 HoistDestinationMap[Orig] = New;
603 DT->addNewBlock(New, HoistTarget);
604 if (CurLoop->getParentLoop())
605 CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
606 ++NumCreatedBlocks;
607 LLVM_DEBUG(dbgs() << "LICM created " << New->getName()
608 << " as hoist destination for " << Orig->getName()
609 << "\n");
610 return New;
611 };
612 BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
613 BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
614 BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);
615
616 // Link up these blocks with branches.
617 if (!HoistCommonSucc->getTerminator()) {
618 // The new common successor we've generated will branch to whatever that
619 // hoist target branched to.
620 BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
621 assert(TargetSucc && "Expected hoist target to have a single successor");
622 HoistCommonSucc->moveBefore(TargetSucc);
623 BranchInst::Create(TargetSucc, HoistCommonSucc);
624 }
625 if (!HoistTrueDest->getTerminator()) {
626 HoistTrueDest->moveBefore(HoistCommonSucc);
627 BranchInst::Create(HoistCommonSucc, HoistTrueDest);
628 }
629 if (!HoistFalseDest->getTerminator()) {
630 HoistFalseDest->moveBefore(HoistCommonSucc);
631 BranchInst::Create(HoistCommonSucc, HoistFalseDest);
632 }
633
634 // If BI is being cloned to what was originally the preheader then
635 // HoistCommonSucc will now be the new preheader.
636 if (HoistTarget == InitialPreheader) {
637 // Phis in the loop header now need to use the new preheader.
638 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
639 // The new preheader dominates the loop header.
640 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
641 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
642 DT->changeImmediateDominator(HeaderNode, PreheaderNode);
643 // The preheader hoist destination is now the new preheader, with the
644 // exception of the hoist destination of this branch.
645 for (auto &Pair : HoistDestinationMap)
646 if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
647 Pair.second = HoistCommonSucc;
648 }
649
650 // Now finally clone BI.
651 ReplaceInstWithInst(
652 HoistTarget->getTerminator(),
653 BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
654 ++NumClonedBranches;
655
656 assert(CurLoop->getLoopPreheader() &&
657 "Hoisting blocks should not have destroyed preheader");
658 return HoistDestinationMap[BB];
659 }
660 };
661
439662 /// Walk the specified region of the CFG (defined by all blocks dominated by
440663 /// the specified block, and that are in the current loop) in depth first
441664 /// order w.r.t the DominatorTree. This allows us to visit definitions before
450673 CurLoop != nullptr && CurAST != nullptr && SafetyInfo != nullptr &&
451674 "Unexpected input to hoistRegion");
452675
453 // We want to visit parents before children. We will enque all the parents
454 // before their children in the worklist and process the worklist in order.
455 SmallVector Worklist = collectChildrenInLoop(N, CurLoop);
456
676 ControlFlowHoister CFH(LI, DT, CurLoop);
677
678 // Keep track of instructions that have been hoisted, as they may need to be
679 // re-hoisted if they end up not dominating all of their uses.
680 SmallVector HoistedInstructions;
681
682 // Record what the original preheader is, as we'll need it later if we need to
683 // re-hoist instructions.
684 BasicBlock *OriginalPreheader = CurLoop->getLoopPreheader();
685
686 // For PHI hoisting to work we need to hoist blocks before their successors.
687 // We can do this by iterating through the blocks in the loop in reverse
688 // post-order.
689 LoopBlocksRPO Worklist(CurLoop);
690 Worklist.perform(LI);
457691 bool Changed = false;
458 for (DomTreeNode *DTN : Worklist) {
459 BasicBlock *BB = DTN->getBlock();
692 for (BasicBlock *BB : Worklist) {
460693 // Only need to process the contents of this block if it is not part of a
461694 // subloop (which would already have been processed).
462695 if (inSubLoop(BB, CurLoop, LI))
482715 // Try hoisting the instruction out to the preheader. We can only do
483716 // this if all of the operands of the instruction are loop invariant and
484717 // if it is safe to hoist the instruction.
485 //
718 // TODO: It may be safe to hoist if we are hoisting to a conditional block
719 // and we have accurately duplicated the control flow from the loop header
720 // to that block.
486721 if (CurLoop->hasLoopInvariantOperands(&I) &&
487722 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
488723 isSafeToExecuteUnconditionally(
489724 I, DT, CurLoop, SafetyInfo, ORE,
490725 CurLoop->getLoopPreheader()->getTerminator())) {
491 hoist(I, DT, CurLoop, SafetyInfo, ORE);
726 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
727 HoistedInstructions.push_back(&I);
492728 Changed = true;
493729 continue;
494730 }
513749 I.replaceAllUsesWith(Product);
514750 eraseInstruction(I, *SafetyInfo, CurAST);
515751
516 hoist(*ReciprocalDivisor, DT, CurLoop, SafetyInfo, ORE);
752 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
753 SafetyInfo, ORE);
754 HoistedInstructions.push_back(ReciprocalDivisor);
517755 Changed = true;
518756 continue;
519757 }
525763 CurLoop->hasLoopInvariantOperands(&I) &&
526764 SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
527765 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) {
528 hoist(I, DT, CurLoop, SafetyInfo, ORE);
766 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
767 HoistedInstructions.push_back(&I);
529768 Changed = true;
530769 continue;
531770 }
532 }
533 }
771
772 if (PHINode *PN = dyn_cast(&I)) {
773 if (CFH.canHoistPHI(PN)) {
774 // Redirect incoming blocks first to ensure that we create hoisted
775 // versions of those blocks before we hoist the phi.
776 for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
777 PN->setIncomingBlock(
778 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
779 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
780 ORE);
781 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected");
782 Changed = true;
783 continue;
784 }
785 }
786
787 // Remember possibly hoistable branches so we can actually hoist them
788 // later if needed.
789 if (BranchInst *BI = dyn_cast(&I))
790 CFH.registerPossiblyHoistableBranch(BI);
791 }
792 }
793
794 // If we hoisted instructions to a conditional block they may not dominate
795 // their uses that weren't hoisted (such as phis where some operands are not
796 // loop invariant). If so make them unconditional by moving them to the end of
797 // the original preheader, which is guaranteed to dominate everything in the
798 // loop. We iterate through the instructions in reverse order which ensures
799 // that when we rehoist an instruction we rehoist its operands.
800 Instruction *HoistPoint = OriginalPreheader->getTerminator();
801 for (Instruction *I : reverse(HoistedInstructions)) {
802 if (!llvm::all_of(I->uses(), [&](Use &U) { return DT->dominates(I, U); })) {
803 LLVM_DEBUG(dbgs() << "LICM rehoisting to " << OriginalPreheader->getName()
804 << ": " << *I << "\n");
805 moveInstructionBefore(*I, *HoistPoint, *SafetyInfo);
806 HoistPoint = I;
807 Changed = true;
808 }
809 }
810
811 // Now that we've finished hoisting make sure that LI and DT are still valid.
812 #ifndef NDEBUG
813 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
814 "Dominator tree verification failed");
815 LI->verify(*DT);
816 #endif
534817
535818 return Changed;
536819 }
10991382 /// is safe to hoist, this instruction is called to do the dirty work.
11001383 ///
11011384 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1102 ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
1103 auto *Preheader = CurLoop->getLoopPreheader();
1104 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Preheader->getName() << ": " << I
1385 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1386 OptimizationRemarkEmitter *ORE) {
1387 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I
11051388 << "\n");
11061389 ORE->emit([&]() {
11071390 return OptimizationRemark(DEBUG_TYPE, "Hoisted", &I) << "hoisting "
11191402 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
11201403 I.dropUnknownNonDebugMetadata();
11211404
1122 // Move the new node to the Preheader, before its terminator.
1123 moveInstructionBefore(I, *Preheader->getTerminator(), *SafetyInfo);
1405 if (isa(I))
1406 // Move the new node to the end of the phi list in the destination block.
1407 moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo);
1408 else
1409 // Move the new node to the destination block, before its terminator.
1410 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo);
11241411
11251412 // Do not retain debug locations when we are moving instructions to different
11261413 // basic blocks, because we want to avoid jumpy line tables. Calls, however,
0 ; RUN: opt -S -licm < %s | FileCheck %s
1 ; RUN: opt -passes='require,loop(licm)' -S < %s | FileCheck %s
2
3 ; CHECK-LABEL: @triangle_phi
4 define void @triangle_phi(i32 %x, i32* %p) {
5 ; CHECK-LABEL: entry:
6 ; CHECK: %cmp1 = icmp sgt i32 %x, 0
7 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
8 entry:
9 br label %loop
10
11 ; CHECK: [[IF_LICM]]:
12 ; CHECK: %add = add i32 %x, 1
13 ; CHECK: br label %[[THEN_LICM]]
14
15 ; CHECK: [[THEN_LICM]]:
16 ; CHECK: phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
17 ; CHECK: store i32 %phi, i32* %p
18 ; CHECK: %cmp2 = icmp ne i32 %phi, 0
19 ; CHECK: br label %loop
20
21 loop:
22 %cmp1 = icmp sgt i32 %x, 0
23 br i1 %cmp1, label %if, label %then
24
25 if:
26 %add = add i32 %x, 1
27 br label %then
28
29 then:
30 %phi = phi i32 [ %add, %if ], [ %x, %loop ]
31 store i32 %phi, i32* %p
32 %cmp2 = icmp ne i32 %phi, 0
33 br i1 %cmp2, label %loop, label %end
34
35 end:
36 ret void
37 }
38
39 ; CHECK-LABEL: @diamond_phi
40 define void @diamond_phi(i32 %x, i32* %p) {
41 ; CHECK-LABEL: entry:
42 ; CHECK: %cmp1 = icmp sgt i32 %x, 0
43 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
44 entry:
45 br label %loop
46
47 ; CHECK: [[IF_LICM]]:
48 ; CHECK: %add = add i32 %x, 1
49 ; CHECK: br label %[[THEN_LICM:.*]]
50
51 ; CHECK: [[ELSE_LICM]]:
52 ; CHECK: %sub = sub i32 %x, 1
53 ; CHECK: br label %[[THEN_LICM]]
54
55 ; CHECK: [[THEN_LICM]]
56 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
57 ; CHECK: store i32 %phi, i32* %p
58 ; CHECK: %cmp2 = icmp ne i32 %phi, 0
59 ; CHECK: br label %loop
60
61 loop:
62 %cmp1 = icmp sgt i32 %x, 0
63 br i1 %cmp1, label %if, label %else
64
65 if:
66 %add = add i32 %x, 1
67 br label %then
68
69 else:
70 %sub = sub i32 %x, 1
71 br label %then
72
73 then:
74 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
75 store i32 %phi, i32* %p
76 %cmp2 = icmp ne i32 %phi, 0
77 br i1 %cmp2, label %loop, label %end
78
79 end:
80 ret void
81 }
82
83 ; TODO: This is currently too complicated for us to be able to hoist the phi.
84 ; CHECK-LABEL: @three_way_phi
85 define void @three_way_phi(i32 %x, i32* %p) {
86 ; CHECK-LABEL: entry:
87 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
88 ; CHECK-DAG: %add = add i32 %x, 1
89 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
90 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
91
92 ; CHECK: [[IF_LICM]]:
93 ; CHECK: br label %[[THEN_LICM:.*]]
94
95 ; CHECK: [[THEN_LICM]]:
96 ; CHECK: %sub = sub i32 %x, 1
97 ; CHECK: br label %loop
98
99 entry:
100 br label %loop
101
102 loop:
103 %cmp1 = icmp sgt i32 %x, 0
104 br i1 %cmp1, label %if, label %then
105
106 if:
107 %add = add i32 %x, 1
108 %cmp2 = icmp sgt i32 %add, 0
109 br i1 %cmp2, label %if.if, label %then
110
111 if.if:
112 %sub = sub i32 %x, 1
113 br label %then
114
115 then:
116 %phi = phi i32 [ 0, %loop ], [ %add, %if ], [ %sub, %if.if ]
117 store i32 %phi, i32* %p
118 %cmp3 = icmp ne i32 %phi, 0
119 br i1 %cmp3, label %loop, label %end
120
121 end:
122 ret void
123 }
124
125 ; TODO: This is currently too complicated for us to be able to hoist the phi.
126 ; CHECK-LABEL: @tree_phi
127 define void @tree_phi(i32 %x, i32* %p) {
128 ; CHECK-LABEL: entry:
129 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
130 ; CHECK-DAG: %add = add i32 %x, 1
131 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
132 ; CHECK-DAG: %sub = sub i32 %x, 1
133 ; CHECK: br label %loop
134
135 entry:
136 br label %loop
137
138 loop:
139 %cmp1 = icmp sgt i32 %x, 0
140 br i1 %cmp1, label %if, label %else
141
142 if:
143 %add = add i32 %x, 1
144 %cmp2 = icmp sgt i32 %add, 0
145 br i1 %cmp2, label %if.if, label %if.else
146
147 if.if:
148 br label %then
149
150 if.else:
151 br label %then
152
153 else:
154 %sub = sub i32 %x, 1
155 br label %then
156
157 then:
158 %phi = phi i32 [ %add, %if.if ], [ 0, %if.else ], [ %sub, %else ]
159 store i32 %phi, i32* %p
160 %cmp3 = icmp ne i32 %phi, 0
161 br i1 %cmp3, label %loop, label %end
162
163 end:
164 ret void
165 }
166
167 ; TODO: We can hoist the first phi, but not the second.
168 ; CHECK-LABEL: @phi_phi
169 define void @phi_phi(i32 %x, i32* %p) {
170 ; CHECK-LABEL: entry:
171 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
172 ; CHECK-DAG: %add = add i32 %x, 1
173 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
174 ; CHECK-DAG: %sub = sub i32 %x, 1
175 ; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
176
177 ; CHECK: [[IF_IF_LICM]]:
178 ; CHECK: br label %[[IF_THEN_LICM:.*]]
179
180 ; CHECK: [[IF_ELSE_LICM]]:
181 ; CHECK: br label %[[IF_THEN_LICM]]
182
183 ; CHECK: [[IF_THEN_LICM]]:
184 ; CHECK: %phi1 = phi i32 [ %add, %[[IF_IF_LICM]] ], [ 0, %[[IF_ELSE_LICM]] ]
185 ; CHECK: br label %loop
186
187 entry:
188 br label %loop
189
190 loop:
191 %cmp1 = icmp sgt i32 %x, 0
192 br i1 %cmp1, label %if, label %else
193
194 if:
195 %add = add i32 %x, 1
196 %cmp2 = icmp sgt i32 %add, 0
197 br i1 %cmp2, label %if.if, label %if.else
198
199 if.if:
200 br label %if.then
201
202 if.else:
203 br label %if.then
204
205 if.then:
206 %phi1 = phi i32 [ %add, %if.if ], [ 0, %if.else ]
207 br label %then
208
209 else:
210 %sub = sub i32 %x, 1
211 br label %then
212
213 then:
214 %phi2 = phi i32 [ %phi1, %if.then ], [ %sub, %else ]
215 store i32 %phi2, i32* %p
216 %cmp3 = icmp ne i32 %phi2, 0
217 br i1 %cmp3, label %loop, label %end
218
219 end:
220 ret void
221 }
222
223 ; Check that we correctly duplicate empty control flow.
224 ; CHECK-LABEL: @empty_triangle_phi
225 define i8 @empty_triangle_phi(i32 %x, i32 %y) {
226 ; CHECK-LABEL: entry:
227 ; CHECK: %cmp1 = icmp eq i32 %x, 0
228 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
229 entry:
230 br label %loop
231
232 ; CHECK: [[IF_LICM]]:
233 ; CHECK: br label %[[THEN_LICM]]
234
235 ; CHECK: [[THEN_LICM]]:
236 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
237 ; CHECK: %cmp2 = icmp eq i32 %y, 0
238 ; CHECK: br label %loop
239
240 loop:
241 %cmp1 = icmp eq i32 %x, 0
242 br i1 %cmp1, label %if, label %then
243
244 if:
245 br label %then
246
247 then:
248 %phi = phi i8 [ 0, %if ], [ 1, %loop ]
249 %cmp2 = icmp eq i32 %y, 0
250 br i1 %cmp2, label %end, label %loop
251
252 end:
253 ret i8 %phi
254 }
255
256 ; CHECK-LABEL: @empty_diamond_phi
257 define i8 @empty_diamond_phi(i32 %x, i32 %y) {
258 ; CHECK-LABEL: entry:
259 ; CHECK: %cmp1 = icmp eq i32 %x, 0
260 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
261 entry:
262 br label %loop
263
264 ; CHECK: [[IF_LICM]]:
265 ; CHECK: br label %[[THEN_LICM:.*]]
266
267 ; CHECK: [[ELSE_LICM]]:
268 ; CHECK: br label %[[THEN_LICM]]
269
270 ; CHECK: [[THEN_LICM]]:
271 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
272 ; CHECK: %cmp2 = icmp eq i32 %y, 0
273 ; CHECK: br label %loop
274
275 loop:
276 %cmp1 = icmp eq i32 %x, 0
277 br i1 %cmp1, label %if, label %else
278
279 if:
280 br label %then
281
282 else:
283 br label %then
284
285 then:
286 %phi = phi i8 [ 0, %if ], [ 1, %else ]
287 %cmp2 = icmp eq i32 %y, 0
288 br i1 %cmp2, label %end, label %loop
289
290 end:
291 ret i8 %phi
292 }
293
294 ; Check that we correctly handle the case that the first thing we try to hoist is a phi.
295 ; CHECK-LABEL: @empty_triangle_phi_first
296 define i8 @empty_triangle_phi_first(i32 %x, i1 %cond) {
297 ; CHECK-LABEL: entry:
298 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
299 entry:
300 br label %loop
301
302 ; CHECK: [[IF_LICM]]:
303 ; CHECK: br label %[[THEN_LICM]]
304
305 ; CHECK: [[THEN_LICM]]:
306 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
307 ; CHECK: %cmp = icmp eq i32 %x, 0
308 ; CHECK: br label %loop
309
310 loop:
311 br i1 %cond, label %if, label %then
312
313 if:
314 br label %then
315
316 then:
317 %phi = phi i8 [ 0, %if ], [ 1, %loop ]
318 %cmp = icmp eq i32 %x, 0
319 br i1 %cmp, label %end, label %loop
320
321 end:
322 ret i8 %phi
323 }
324
325 ; CHECK-LABEL: @empty_diamond_phi
326 define i8 @empty_diamond_phi_first(i32 %x, i1 %cond) {
327 ; CHECK-LABEL: entry:
328 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
329 entry:
330 br label %loop
331
332 ; CHECK: [[IF_LICM]]:
333 ; CHECK: br label %[[THEN_LICM:.*]]
334
335 ; CHECK: [[ELSE_LICM]]:
336 ; CHECK: br label %[[THEN_LICM]]
337
338 ; CHECK: [[THEN_LICM]]:
339 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
340 ; CHECK: %cmp = icmp eq i32 %x, 0
341 ; CHECK: br label %loop
342
343 loop:
344 br i1 %cond, label %if, label %else
345
346 if:
347 br label %then
348
349 else:
350 br label %then
351
352 then:
353 %phi = phi i8 [ 0, %if ], [ 1, %else ]
354 %cmp = icmp eq i32 %x, 0
355 br i1 %cmp, label %end, label %loop
356
357 end:
358 ret i8 %phi
359 }
360
361 ; CHECK-LABEL: @empty_triangle_phi_first
362 define i8 @empty_triangle_phi_first_empty_loop_head(i32 %x, i1 %cond) {
363 ; CHECK-LABEL: entry:
364 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
365 entry:
366 br label %loop
367
368 ; CHECK: [[IF_LICM]]:
369 ; CHECK: br label %[[THEN_LICM]]
370
371 ; CHECK: [[THEN_LICM]]:
372 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
373 ; CHECK: %cmp = icmp eq i32 %x, 0
374 ; CHECK: br label %loop
375
376 loop:
377 br label %test
378
379 test:
380 br i1 %cond, label %if, label %then
381
382 if:
383 br label %then
384
385 then:
386 %phi = phi i8 [ 0, %if ], [ 1, %test ]
387 %cmp = icmp eq i32 %x, 0
388 br i1 %cmp, label %end, label %loop
389
390 end:
391 ret i8 %phi
392 }
393
394 ; CHECK-LABEL: @empty_diamond_phi_first_empty_loop_head
395 define i8 @empty_diamond_phi_first_empty_loop_head(i32 %x, i1 %cond) {
396 ; CHECK-LABEL: entry:
397 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
398 entry:
399 br label %loop
400
401 ; CHECK: [[IF_LICM]]:
402 ; CHECK: br label %[[THEN_LICM:.*]]
403
404 ; CHECK: [[ELSE_LICM]]:
405 ; CHECK: br label %[[THEN_LICM]]
406
407 ; CHECK: [[THEN_LICM]]:
408 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
409 ; CHECK: %cmp = icmp eq i32 %x, 0
410 ; CHECK: br label %loop
411
412 loop:
413 br label %test
414
415 test:
416 br i1 %cond, label %if, label %else
417
418 if:
419 br label %then
420
421 else:
422 br label %then
423
424 then:
425 %phi = phi i8 [ 0, %if ], [ 1, %else ]
426 %cmp = icmp eq i32 %x, 0
427 br i1 %cmp, label %end, label %loop
428
429 end:
430 ret i8 %phi
431 }
432
433 ; The phi is on one branch of a diamond while simultaneously at the end of a
434 ; triangle. Check that we duplicate the triangle and not the diamond.
435 ; CHECK-LABEL: @triangle_diamond
436 define void @triangle_diamond(i32* %ptr, i32 %x, i32 %y) {
437 ; CHECK-LABEL: entry:
438 ; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
439 ; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
440 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
441 entry:
442 br label %loop
443
444 ; CHECK: [[IF_LICM]]:
445 ; CHECK: br label %[[THEN_LICM]]
446
447 ; CHECK: [[THEN_LICM]]:
448 ; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
449
450 loop:
451 %cmp1 = icmp ne i32 %x, 0
452 br i1 %cmp1, label %if, label %then
453
454 if:
455 %cmp2 = icmp ne i32 %y, 0
456 br i1 %cmp2, label %if.then, label %then
457
458 then:
459 %phi = phi i32 [ 0, %if ], [ 127, %loop ]
460 store i32 %phi, i32* %ptr
461 br label %end
462
463 if.then:
464 br label %end
465
466 end:
467 br label %loop
468 }
469
470 ; As the previous, but the end of the diamond is the head of the loop.
471 ; CHECK-LABEL: @triangle_diamond_backedge
472 define void @triangle_diamond_backedge(i32* %ptr, i32 %x, i32 %y) {
473 ; CHECK-LABEL: entry:
474 ; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
475 ; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
476 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
477 entry:
478 br label %loop
479
480 ; CHECK: [[IF_LICM]]:
481 ; CHECK: br label %[[THEN_LICM]]
482
483 ; CHECK: [[THEN_LICM]]:
484 ; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
485
486 loop:
487 %cmp1 = icmp ne i32 %x, 0
488 br i1 %cmp1, label %if, label %then
489
490 if:
491 %cmp2 = icmp ne i32 %y, 0
492 br i1 %cmp2, label %backedge, label %then
493
494 then:
495 %phi = phi i32 [ 0, %if ], [ 127, %loop ]
496 store i32 %phi, i32* %ptr
497 br label %loop
498
499 backedge:
500 br label %loop
501 }
502
503 ; TODO: The inner diamonds can be hoisted, but not currently the outer diamond
504 ; CHECK-LABEL: @diamonds_inside_diamond
505 define void @diamonds_inside_diamond(i32 %x, i32* %p) {
506 ; CHECK-LABEL: entry:
507 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
508 ; CHECK-DAG: %cmp3 = icmp slt i32 %x, -10
509 ; CHECK: br i1 %cmp3, label %[[ELSE_IF_LICM:.*]], label %[[ELSE_ELSE_LICM:.*]]
510 entry:
511 br label %loop
512
513 ; CHECK: [[ELSE_IF_LICM]]:
514 ; CHECK: br label %[[ELSE_THEN_LICM:.*]]
515
516 ; CHECK: [[ELSE_ELSE_LICM]]:
517 ; CHECK: br label %[[ELSE_THEN_LICM]]
518
519 ; CHECK: [[ELSE_THEN_LICM]]:
520 ; CHECK: %phi2 = phi i32 [ 2, %[[ELSE_IF_LICM]] ], [ 3, %[[ELSE_ELSE_LICM]] ]
521 ; CHECK: %cmp2 = icmp sgt i32 %x, 10
522 ; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
523
524 ; CHECK: [[IF_IF_LICM]]:
525 ; CHECK: br label %[[IF_THEN_LICM:.*]]
526
527 ; CHECK: [[IF_ELSE_LICM]]:
528 ; CHECK: br label %[[IF_THEN_LICM]]
529
530 ; CHECK: [[IF_THEN_LICM]]:
531 ; CHECK: %phi1 = phi i32 [ 0, %[[IF_IF_LICM]] ], [ 1, %[[IF_ELSE_LICM]] ]
532 ; CHECK: br label %loop
533
534 loop:
535 %cmp1 = icmp sgt i32 %x, 0
536 br i1 %cmp1, label %if, label %else
537
538 if:
539 %cmp2 = icmp sgt i32 %x, 10
540 br i1 %cmp2, label %if.if, label %if.else
541
542 if.if:
543 br label %if.then
544
545 if.else:
546 br label %if.then
547
548 if.then:
549 %phi1 = phi i32 [ 0, %if.if ], [ 1, %if.else ]
550 br label %then
551
552 else:
553 %cmp3 = icmp slt i32 %x, -10
554 br i1 %cmp3, label %else.if, label %else.else
555
556 else.if:
557 br label %else.then
558
559 else.else:
560 br label %else.then
561
562 else.then:
563 %phi2 = phi i32 [ 2, %else.if ], [ 3, %else.else ]
564 br label %then
565
566 then:
567 %phi3 = phi i32 [ %phi1, %if.then ], [ %phi2, %else.then ]
568 store i32 %phi3, i32* %p
569 %cmp4 = icmp ne i32 %phi3, 0
570 br i1 %cmp4, label %loop, label %end
571
572 end:
573 ret void
574 }
575
576 ; We can hoist blocks that contain an edge that exits the loop by ignoring that
577 ; edge in the hoisted block.
578 ; CHECK-LABEL: @triangle_phi_loopexit
579 define void @triangle_phi_loopexit(i32 %x, i32* %p) {
580 ; CHECK-LABEL: entry:
581 ; CHECK-DAG: %add = add i32 %x, 1
582 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
583 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
584 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
585 entry:
586 br label %loop
587
588 ; CHECK: [[IF_LICM]]:
589 ; CHECK: br label %[[THEN_LICM]]
590
591 ; CHECK: [[THEN_LICM]]:
592 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
593
594 loop:
595 %cmp1 = icmp sgt i32 %x, 0
596 br i1 %cmp1, label %if, label %then
597
598 if:
599 %add = add i32 %x, 1
600 %cmp2 = icmp sgt i32 10, %add
601 br i1 %cmp2, label %then, label %end
602
603 then:
604 %phi = phi i32 [ %add, %if ], [ %x, %loop ]
605 store i32 %phi, i32* %p
606 %cmp3 = icmp ne i32 %phi, 0
607 br i1 %cmp3, label %loop, label %end
608
609 end:
610 ret void
611 }
612
613 ; CHECK-LABEL: @diamond_phi_oneloopexit
614 define void @diamond_phi_oneloopexit(i32 %x, i32* %p) {
615 ; CHECK-LABEL: entry:
616 ; CHECK-DAG: %add = add i32 %x, 1
617 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
618 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
619 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
620 entry:
621 br label %loop
622
623 ; CHECK: [[IF_LICM]]:
624 ; CHECK: br label %[[THEN_LICM:.*]]
625
626 ; CHECK: [[ELSE_LICM]]:
627 ; CHECK: %sub = sub i32 %x, 1
628 ; CHECK: br label %[[THEN_LICM]]
629
630 ; CHECK: [[THEN_LICM]]
631 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
632 ; CHECK: %cmp3 = icmp ne i32 %phi, 0
633 ; CHECK: br label %loop
634
635 loop:
636 %cmp1 = icmp sgt i32 %x, 0
637 br i1 %cmp1, label %if, label %else
638
639 if:
640 %add = add i32 %x, 1
641 %cmp2 = icmp sgt i32 10, %add
642 br i1 %cmp2, label %then, label %end
643
644 else:
645 %sub = sub i32 %x, 1
646 br label %then
647
648 then:
649 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
650 store i32 %phi, i32* %p
651 %cmp3 = icmp ne i32 %phi, 0
652 br i1 %cmp3, label %loop, label %end
653
654 end:
655 ret void
656 }
657
658 ; CHECK-LABEL: @diamond_phi_twoloopexit
659 define void @diamond_phi_twoloopexit(i32 %x, i32* %p) {
660 ; CHECK-LABEL: entry:
661 ; CHECK-DAG: %sub = sub i32 %x, 1
662 ; CHECK-DAG: %add = add i32 %x, 1
663 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
664 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
665 ; CHECK-DAG: %cmp3 = icmp sgt i32 10, %sub
666 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
667 entry:
668 br label %loop
669
670 ; CHECK: [[IF_LICM]]:
671 ; CHECK: br label %[[THEN_LICM:.*]]
672
673 ; CHECK: [[ELSE_LICM]]:
674 ; CHECK: br label %[[THEN_LICM]]
675
676 ; CHECK: [[THEN_LICM]]
677 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
678 ; CHECK: %cmp4 = icmp ne i32 %phi, 0
679 ; CHECK: br label %loop
680
681 loop:
682 %cmp1 = icmp sgt i32 %x, 0
683 br i1 %cmp1, label %if, label %else
684
685 if:
686 %add = add i32 %x, 1
687 %cmp2 = icmp sgt i32 10, %add
688 br i1 %cmp2, label %then, label %end
689
690 else:
691 %sub = sub i32 %x, 1
692 %cmp3 = icmp sgt i32 10, %sub
693 br i1 %cmp3, label %then, label %end
694
695 then:
696 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
697 store i32 %phi, i32* %p
698 %cmp4 = icmp ne i32 %phi, 0
699 br i1 %cmp4, label %loop, label %end
700
701 end:
702 ret void
703 }
704
705 ; The store cannot be hoisted, so add and shr cannot be hoisted into a
706 ; conditional block.
707 ; CHECK-LABEL: @conditional_use
708 define void @conditional_use(i32 %x, i32* %p) {
709 ; CHECK-LABEL: entry:
710 ; CHECK-DAG: %cond = icmp ugt i32 %x, 0
711 ; CHECK-DAG: %add = add i32 %x, 5
712 ; CHECK-DAG: %shr = ashr i32 %add, 1
713 ; CHECK: br label %loop
714 entry:
715 br label %loop
716
717 loop:
718 %cond = icmp ugt i32 %x, 0
719 br i1 %cond, label %if, label %else
720
721 ; CHECK-LABEL: if:
722 ; CHECK: store i32 %shr, i32* %p, align 4
723 if:
724 %add = add i32 %x, 5
725 %shr = ashr i32 %add, 1
726 store i32 %shr, i32* %p, align 4
727 br label %then
728
729 else:
730 br label %then
731
732 then:
733 br label %loop
734 }
735
736 ; A diamond with two triangles on the left and one on the right. This test is
737 ; to check that we have a unique loop preheader when we hoist the store (and so
738 ; don't fail an assertion).
739 ; CHECK-LABEL: @triangles_in_diamond
740 define void @triangles_in_diamond(i32* %ptr) {
741 ; CHECK-LABEL: entry:
742 ; CHECK: store i32 0, i32* %ptr, align 4
743 ; CHECK: br label %loop
744 entry:
745 br label %loop
746
747 loop:
748 br i1 undef, label %left_triangle_1, label %right_triangle
749
750 left_triangle_1:
751 br i1 undef, label %left_triangle_1_if, label %left_triangle_2
752
753 left_triangle_1_if:
754 br label %left_triangle_2
755
756 left_triangle_2:
757 br i1 undef, label %left_triangle_2_if, label %left_triangle_2_then
758
759 left_triangle_2_if:
760 br label %left_triangle_2_then
761
762 left_triangle_2_then:
763 br label %loop.end
764
765 right_triangle:
766 br i1 undef, label %right_triangle.if, label %right_triangle.then
767
768 right_triangle.if:
769 br label %right_triangle.then
770
771 right_triangle.then:
772 br label %loop.end
773
774 loop.end:
775 store i32 0, i32* %ptr, align 4
776 br label %loop
777 }
778
779 ; %cmp dominates its used after being hoisted, but not after %brmerge is rehoisted
780 ; CHECK-LABEL: @rehoist
781 define void @rehoist(i8* %this, i32 %x) {
782 ; CHECK-LABEL: entry:
783 ; CHECK-DAG: %sub = add nsw i32 %x, -1
784 ; CHECK-DAG: %fptr = bitcast i8* %this to void (i8*)*
785 ; CHECK-DAG: %cmp = icmp eq i32 0, %sub
786 ; CHECK-DAG: %brmerge = or i1 %cmp, true
787 entry:
788 %sub = add nsw i32 %x, -1
789 br label %loop
790
791 loop:
792 br i1 undef, label %if1, label %else1
793
794 if1:
795 %fptr = bitcast i8* %this to void (i8*)*
796 call void %fptr(i8* %this)
797 br label %then1
798
799 else1:
800 br label %then1
801
802 then1:
803 %cmp = icmp eq i32 0, %sub
804 br i1 %cmp, label %end, label %else2
805
806 else2:
807 %brmerge = or i1 %cmp, true
808 br i1 %brmerge, label %if3, label %end
809
810 if3:
811 br label %end
812
813 end:
814 br label %loop
815 }
816
817 ; A test case that uses empty blocks in a way that can cause control flow
818 ; hoisting to get confused.
819 ; CHECK-LABEL: @empty_blocks_multiple_conditional_branches
820 define void @empty_blocks_multiple_conditional_branches(float %arg, float* %ptr) {
821 ; CHECK-LABEL: entry
822 ; CHECK-DAG: %div1 = fmul float %arg, 4.000000e+00
823 ; CHECK-DAG: %div2 = fmul float %arg, 2.000000e+00
824 entry:
825 br label %loop
826
827 ; The exact path to the phi isn't checked here, because it depends on whether
828 ; cond2 or cond3 is hoisted first
829 ; CHECK: %phi = phi float [ 0.000000e+00, %{{.*}} ], [ %div1, %{{.*}} ]
830 ; CHECK: br label %loop
831
832 loop:
833 br i1 undef, label %backedge2, label %cond1
834
835 cond1:
836 br i1 undef, label %cond1.if, label %cond1.else
837
838 cond1.else:
839 br label %cond3
840
841 cond1.if:
842 br label %cond1.if.next
843
844 cond1.if.next:
845 br label %cond2
846
847 cond2:
848 %div1 = fmul float %arg, 4.000000e+00
849 br i1 undef, label %cond2.if, label %cond2.then
850
851 cond2.if:
852 br label %cond2.then
853
854 cond2.then:
855 %phi = phi float [ 0.000000e+00, %cond2 ], [ %div1, %cond2.if ]
856 store float %phi, float* %ptr
857 br label %backedge2
858
859 cond3:
860 br i1 undef, label %cond3.then, label %cond3.if
861
862 cond3.if:
863 %div2 = fmul float %arg, 2.000000e+00
864 store float %div2, float* %ptr
865 br label %cond3.then
866
867 cond3.then:
868 br label %loop
869
870 backedge2:
871 br label %loop
872 }
873
874 ; We can't do much here, so mainly just check that we don't crash.
875 ; CHECK-LABEL: @many_path_phi
876 define void @many_path_phi(i32* %ptr1, i32* %ptr2) {
877 ; CHECK-LABEL: entry:
878 ; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
879 ; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
880 ; CHECK: br label %loop
881 entry:
882 br label %loop
883
884 loop:
885 %phi1 = phi i32 [ 0, %entry ], [ %phi2, %end ]
886 %cmp1 = icmp ugt i32 %phi1, 3
887 br i1 %cmp1, label %cond2, label %cond1
888
889 cond1:
890 br i1 undef, label %end, label %cond1.else
891
892 cond1.else:
893 %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
894 %val2 = load i32, i32* %gep2, align 4
895 %cmp2 = icmp eq i32 %val2, 13
896 br i1 %cmp2, label %cond1.end, label %end
897
898 cond1.end:
899 br label %end
900
901 cond2:
902 br i1 undef, label %end, label %cond2.else
903
904 cond2.else:
905 %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
906 %val3 = load i32, i32* %gep3, align 4
907 %cmp3 = icmp eq i32 %val3, 13
908 br i1 %cmp3, label %cond2.end, label %end
909
910 cond2.end:
911 br label %end
912
913 end:
914 %phi2 = phi i32 [ 1, %cond1 ], [ 2, %cond1.else ], [ 3, %cond1.end ], [ 4, %cond2 ], [ 5, %cond2.else ], [ 6, %cond2.end ]
915 br label %loop
916 }
917
918 ; Check that we correctly handle the hoisting of %gep when theres a critical
919 ; edge that branches to the preheader.
920 ; CHECK-LABEL: @crit_edge
921 define void @crit_edge(i32* %ptr, i32 %idx, i1 %cond1, i1 %cond2) {
922 ; CHECK-LABEL: entry:
923 ; CHECK: %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
924 ; CHECK: br label %preheader
925 entry:
926 br label %preheader
927
928 preheader:
929 br label %loop
930
931 loop:
932 br i1 %cond1, label %then, label %if
933
934 if:
935 %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
936 %val = load i32, i32* %gep
937 br label %then
938
939 then:
940 %phi = phi i32 [ %val, %if ], [ 0, %loop ]
941 store i32 %phi, i32* %ptr
942 br i1 %cond2, label %loop, label %crit_edge
943
944 crit_edge:
945 br label %preheader
946 }
947
948 ; Check that the conditional sub is correctly hoisted from the inner loop to the
949 ; preheader of the outer loop.
950 ; CHECK-LABEL: @hoist_from_innermost_loop
951 define void @hoist_from_innermost_loop(i32 %nx, i32* %ptr) {
952 ; CHECK-LABEL: entry:
953 ; CHECK-DAG: %sub = sub nsw i32 0, %nx
954 ; CHECK: br label %outer_loop
955 entry:
956 br label %outer_loop
957
958 outer_loop:
959 br label %middle_loop
960
961 middle_loop:
962 br label %inner_loop
963
964 inner_loop:
965 br i1 undef, label %inner_loop_end, label %if
966
967 if:
968 %sub = sub nsw i32 0, %nx
969 store i32 %sub, i32* %ptr, align 4
970 br label %inner_loop_end
971
972 inner_loop_end:
973 br i1 undef, label %inner_loop, label %middle_loop_end
974
975 middle_loop_end:
976 br i1 undef, label %middle_loop, label %outer_loop_end
977
978 outer_loop_end:
979 br label %outer_loop
980 }
981
982 ; We have a diamond starting from %if, but %if.if is also reachable from %loop,
983 ; so %gep should not be conditionally hoisted.
984 ; CHECK-LABEL: @diamond_with_extra_in_edge
985 define void @diamond_with_extra_in_edge(i32* %ptr1, i32* %ptr2, i32 %arg) {
986 ; CHECK-LABEL: entry:
987 ; CHECK-DAG: %cmp2 = icmp ne i32 0, %arg
988 ; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
989 ; CHECK: br label %loop
990 entry:
991 br label %loop
992
993 loop:
994 %phi1 = phi i32 [ 0, %entry ], [ %phi2, %then ]
995 %cmp1 = icmp ugt i32 16, %phi1
996 br i1 %cmp1, label %if, label %if.if
997
998 if:
999 %cmp2 = icmp ne i32 0, %arg
1000 br i1 %cmp2, label %if.if, label %if.else
1001
1002 if.if:
1003 %gep = getelementptr i32, i32* %ptr1, i32 4
1004 %val = load i32, i32* %gep, align 4
1005 br label %then
1006
1007 if.else:
1008 br label %then
1009
1010 then:
1011 %phi2 = phi i32 [ %val, %if.if ], [ %phi1, %if.else ]
1012 store i32 %phi2, i32* %ptr2, align 4
1013 br label %loop
1014 }
1015
1016 ; %loop/%if/%then form a triangle, but %loop/%if/%then/%end also form a diamond.
1017 ; The triangle should be picked for conditional hoisting.
1018 ; CHECK-LABEL: @both_triangle_and_diamond
1019 define void @both_triangle_and_diamond(i32* %ptr1, i32* %ptr2, i32 %arg) {
1020 ; CHECK-LABEL: entry:
1021 ; CHECK-DAG: %cmp1 = icmp ne i32 0, %arg
1022 ; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
1023 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
1024 entry:
1025 br label %loop
1026
1027 ; CHECK: [[IF_LICM]]:
1028 ; CHECK: br label %[[THEN_LICM]]
1029
1030 ; CHECK: [[THEN_LICM]]:
1031 ; CHECK: %phi2 = phi i32 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
1032 ; CHECK: br label %loop
1033
1034 loop:
1035 %phi1 = phi i32 [ 0, %entry ], [ %phi3, %end ]
1036 %cmp1 = icmp ne i32 0, %arg
1037 br i1 %cmp1, label %if, label %then
1038
1039 if:
1040 %gep = getelementptr i32, i32* %ptr1, i32 4
1041 %val = load i32, i32* %gep, align 4
1042 %cmp2 = icmp ugt i32 16, %phi1
1043 br i1 %cmp2, label %end, label %then
1044
1045 then:
1046 %phi2 = phi i32 [ 0, %if ], [ 1, %loop ]
1047 br label %end
1048
1049 end:
1050 %phi3 = phi i32 [ %phi2, %then ], [ %val, %if ]
1051 store i32 %phi3, i32* %ptr2, align 4
1052 br label %loop
1053 }
1054
1055 ; We shouldn't duplicate the branch at the end of %loop and should instead hoist
1056 ; %val to %entry.
1057 ; CHECK-LABEL: @same_destination_branch
1058 define i32 @same_destination_branch(i32 %arg1, i32 %arg2) {
1059 ; CHECK-LABEL: entry:
1060 ; CHECK-DAG: %cmp1 = icmp ne i32 %arg2, 0
1061 ; CHECK-DAG: %val = add i32 %arg1, 1
1062 ; CHECK: br label %loop
1063 entry:
1064 br label %loop
1065
1066 loop:
1067 %phi = phi i32 [ 0, %entry ], [ %add, %then ]
1068 %add = add i32 %phi, 1
1069 %cmp1 = icmp ne i32 %arg2, 0
1070 br i1 %cmp1, label %if, label %if
1071
1072 if:
1073 %val = add i32 %arg1, 1
1074 br label %then
1075
1076 then:
1077 %cmp2 = icmp ne i32 %val, %phi
1078 br i1 %cmp2, label %loop, label %end
1079
1080 end:
1081 ret i32 %val
1082 }
1083
1084 ; Diamond-like control flow but the left/right blocks actually have the same
1085 ; destinations.
1086 ; TODO: We could potentially hoist all of phi2-4, but currently only hoist phi2.
1087 ; CHECK-LABEL: @diamond_like_same_destinations
1088 define i32 @diamond_like_same_destinations(i32 %arg1, i32 %arg2) {
1089 ; CHECK-LABEL: entry:
1090 ; CHECK-DAG: %cmp1 = icmp ne i32 %arg1, 0
1091 ; CHECK-DAG: %cmp2 = icmp ugt i32 %arg2, 1
1092 ; CHECK-DAG: %cmp3 = icmp ugt i32 %arg2, 2
1093 ; CHECK: br i1 %cmp1, label %[[LEFT1_LICM:.*]], label %[[RIGHT1_LICM:.*]]
1094 entry:
1095 br label %loop
1096
1097 ; CHECK: [[LEFT1_LICM]]:
1098 ; CHECK: br label %[[LEFT2_LICM:.*]]
1099
1100 ; CHECK: [[RIGHT1_LICM]]:
1101 ; CHECK: br label %[[LEFT2_LICM]]
1102
1103 ; CHECK: [[LEFT2_LICM]]:
1104 ; CHECK: %phi2 = phi i32 [ 0, %[[LEFT1_LICM]] ], [ 1, %[[RIGHT1_LICM]] ]
1105 ; CHECK: br label %loop
1106
1107 loop:
1108 %phi1 = phi i32 [ 0, %entry ], [ %add, %loopend ]
1109 %add = add i32 %phi1, 1
1110 %cmp1 = icmp ne i32 %arg1, 0
1111 br i1 %cmp1, label %left1, label %right1
1112
1113 left1:
1114 %cmp2 = icmp ugt i32 %arg2, 1
1115 br i1 %cmp2, label %left2, label %right2
1116
1117 right1:
1118 %cmp3 = icmp ugt i32 %arg2, 2
1119 br i1 %cmp3, label %left2, label %right2
1120
1121 left2:
1122 %phi2 = phi i32 [ 0, %left1 ], [ 1, %right1 ]
1123 br label %loopend
1124
1125 right2:
1126 %phi3 = phi i32 [ 2, %left1 ], [ 3, %right1 ]
1127 br label %loopend
1128
1129 loopend:
1130 %phi4 = phi i32 [ %phi2, %left2 ], [ %phi3, %right2 ]
1131 %cmp4 = icmp ne i32 %phi1, 32
1132 br i1 %cmp4, label %loop, label %end
1133
1134 end:
1135 ret i32 %phi4
1136 }
1137
1138 ; A phi with multiple incoming values for the same block due to a branch with
1139 ; two destinations that are actually the same. We can't hoist this.
1140 ; TODO: This could be hoisted by erasing one of the incoming values.
1141 ; CHECK-LABEL: @phi_multiple_values_same_block
1142 define i32 @phi_multiple_values_same_block(i32 %arg) {
1143 ; CHECK-LABEL: entry:
1144 ; CHECK: %cmp = icmp sgt i32 %arg, 4
1145 ; CHECK-NOT: phi
1146 ; CHECK: br label %loop
1147 entry:
1148 br label %loop
1149
1150 loop:
1151 %cmp = icmp sgt i32 %arg, 4
1152 br i1 %cmp, label %if, label %then
1153
1154 if:
1155 br i1 undef, label %then, label %then
1156
1157 then:
1158 %phi = phi i32 [ %arg, %loop ], [ 1, %if ], [ 1, %if ]
1159 br i1 undef, label %exit, label %loop
1160
1161 exit:
1162 ret i32 %phi
1163 }
265265 ; variant/invariant values being stored to invariant address.
266266 ; test checks that the last element of the phi is extracted and scalar stored
267267 ; into the uniform address within the loop.
268 ; Since the condition and the phi is loop invariant, they are LICM'ed after
268 ; Since the condition and the phi is loop invariant, they are LICM'ed before
269269 ; vectorization.
270270 ; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv
271271 ; CHECK-NEXT: entry:
272 ; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
273 ; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
272274 ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
273275 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]]
276 ; CHECK-NEXT: br i1 [[CMP]], label %[[COND_STORE_LICM:.*]], label %[[COND_STORE_K_LICM:.*]]
277 ; CHECK: [[COND_STORE_LICM]]:
278 ; CHECK-NEXT: br label %[[LATCH_LICM:.*]]
279 ; CHECK: [[COND_STORE_K_LICM]]:
280 ; CHECK-NEXT: br label %[[LATCH_LICM]]
281 ; CHECK: [[LATCH_LICM]]:
282 ; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], %[[COND_STORE_LICM]] ], [ [[K]], %[[COND_STORE_K_LICM]] ]
274283 ; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
275284 ; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
276285 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
277286 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
278287 ; CHECK: vector.memcheck:
279 ; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
280 ; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
281288 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
282289 ; CHECK-NEXT: [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
283290 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
290297 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
291298 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[NTRUNC]], i32 0
292299 ; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer
293 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i32 3
294 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[K]], i32 3
295 ; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[TMP3]]
296 ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
297300 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
298301 ; CHECK: vector.body:
299302 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
300303 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
301304 ; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
302305 ; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32>* [[TMP7]], align 4
303 ; CHECK-NEXT: store i32 [[TMP5]], i32* [[A]], align 4
306 ; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4
304307 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
305308 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
306309 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
320323 ; CHECK: cond_store_k:
321324 ; CHECK-NEXT: br label [[LATCH]]
322325 ; CHECK: latch:
323 ; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ]
324326 ; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4
325327 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
326328 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]