llvm.org GIT mirror llvm / e027060
Revert "[LICM] Make LICM able to hoist phis" This reverts commit r347190. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@347225 91177308-0d34-0410-b5e6-96231b3b80d8 Benjamin Kramer 10 months ago
3 changed file(s) with 27 addition(s) and 1480 deletion(s). Raw diff Collapse all Expand all
3030 //===----------------------------------------------------------------------===//
3131
3232 #include "llvm/Transforms/Scalar/LICM.h"
33 #include "llvm/ADT/SetOperations.h"
3433 #include "llvm/ADT/Statistic.h"
3534 #include "llvm/Analysis/AliasAnalysis.h"
3635 #include "llvm/Analysis/AliasSetTracker.h"
4140 #include "llvm/Analysis/GuardUtils.h"
4241 #include "llvm/Analysis/Loads.h"
4342 #include "llvm/Analysis/LoopInfo.h"
44 #include "llvm/Analysis/LoopIterator.h"
4543 #include "llvm/Analysis/LoopPass.h"
4644 #include "llvm/Analysis/MemoryBuiltins.h"
4745 #include "llvm/Analysis/MemorySSA.h"
7674
7775 #define DEBUG_TYPE "licm"
7876
79 STATISTIC(NumCreatedBlocks, "Number of blocks created");
80 STATISTIC(NumClonedBranches, "Number of branches cloned");
8177 STATISTIC(NumSunk, "Number of instructions sunk out of loop");
8278 STATISTIC(NumHoisted, "Number of instructions hoisted out of loop");
8379 STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk");
106102 const LoopSafetyInfo *SafetyInfo,
107103 TargetTransformInfo *TTI, bool &FreeInLoop);
108104 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
109 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
105 ICFLoopSafetyInfo *SafetyInfo,
110106 OptimizationRemarkEmitter *ORE);
111107 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
112108 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
440436 return Changed;
441437 }
442438
443 // This is a helper class for hoistRegion to make it able to hoist control flow
444 // in order to be able to hoist phis. The way this works is that we initially
445 // start hoisting to the loop preheader, and when we see a loop invariant branch
446 // we make note of this. When we then come to hoist an instruction that's
447 // conditional on such a branch we duplicate the branch and the relevant control
448 // flow, then hoist the instruction into the block corresponding to its original
449 // block in the duplicated control flow.
450 class ControlFlowHoister {
451 private:
452 // Information about the loop we are hoisting from
453 LoopInfo *LI;
454 DominatorTree *DT;
455 Loop *CurLoop;
456
457 // A map of blocks in the loop to the block their instructions will be hoisted
458 // to.
459 DenseMap HoistDestinationMap;
460
461 // The branches that we can hoist, mapped to the block that marks a
462 // convergence point of their control flow.
463 DenseMap HoistableBranches;
464
465 public:
466 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop)
467 : LI(LI), DT(DT), CurLoop(CurLoop) {}
468
469 void registerPossiblyHoistableBranch(BranchInst *BI) {
470 // We can only hoist conditional branches with loop invariant operands.
471 if (!BI->isConditional() || !CurLoop->hasLoopInvariantOperands(BI))
472 return;
473
474 // The branch destinations need to be in the loop, and we don't gain
475 // anything by duplicating conditional branches with duplicate successors,
476 // as it's essentially the same as an unconditional branch.
477 BasicBlock *TrueDest = BI->getSuccessor(0);
478 BasicBlock *FalseDest = BI->getSuccessor(1);
479 if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
480 TrueDest == FalseDest)
481 return;
482
483 // We can hoist BI if one branch destination is the successor of the other,
484 // or both have common successor which we check by seeing if the
485 // intersection of their successors is non-empty.
486 // TODO: This could be expanded to allowing branches where both ends
487 // eventually converge to a single block.
488 SmallPtrSet TrueDestSucc, FalseDestSucc;
489 TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
490 FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
491 BasicBlock *CommonSucc = nullptr;
492 if (TrueDestSucc.count(FalseDest)) {
493 CommonSucc = FalseDest;
494 } else if (FalseDestSucc.count(TrueDest)) {
495 CommonSucc = TrueDest;
496 } else {
497 set_intersect(TrueDestSucc, FalseDestSucc);
498 // If there's one common successor use that.
499 if (TrueDestSucc.size() == 1)
500 CommonSucc = *TrueDestSucc.begin();
501 // If there's more than one pick whichever appears first in the block list
502 // (we can't use the value returned by TrueDestSucc.begin() as it's
503 // unpredicatable which element gets returned).
504 else if (!TrueDestSucc.empty()) {
505 Function *F = TrueDest->getParent();
506 auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
507 auto It = std::find_if(F->begin(), F->end(), IsSucc);
508 assert(It != F->end() && "Could not find successor in function");
509 CommonSucc = &*It;
510 }
511 }
512 // The common successor has to be dominated by the branch, as otherwise
513 // there will be some other path to the successor that will not be
514 // controlled by this branch so any phi we hoist would be controlled by the
515 // wrong condition. This also takes care of avoiding hoisting of loop back
516 // edges.
517 // TODO: In some cases this could be relaxed if the successor is dominated
518 // by another block that's been hoisted and we can guarantee that the
519 // control flow has been replicated exactly.
520 if (CommonSucc && DT->dominates(BI, CommonSucc))
521 HoistableBranches[BI] = CommonSucc;
522 }
523
524 bool canHoistPHI(PHINode *PN) {
525 // The phi must have loop invariant operands.
526 if (!CurLoop->hasLoopInvariantOperands(PN))
527 return false;
528 // We can hoist phis if the block they are in is the target of hoistable
529 // branches which cover all of the predecessors of the block.
530 SmallPtrSet PredecessorBlocks;
531 BasicBlock *BB = PN->getParent();
532 for (BasicBlock *PredBB : predecessors(BB))
533 PredecessorBlocks.insert(PredBB);
534 // If we have less predecessor blocks than predecessors then the phi will
535 // have more than one incoming value for the same block which we can't
536 // handle.
537 // TODO: This could be handled be erasing some of the duplicate incoming
538 // values.
539 if (PredecessorBlocks.size() != pred_size(BB))
540 return false;
541 for (auto &Pair : HoistableBranches) {
542 if (Pair.second == BB) {
543 // Which blocks are predecessors via this branch depends on if the
544 // branch is triangle-like or diamond-like.
545 if (Pair.first->getSuccessor(0) == BB) {
546 PredecessorBlocks.erase(Pair.first->getParent());
547 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
548 } else if (Pair.first->getSuccessor(1) == BB) {
549 PredecessorBlocks.erase(Pair.first->getParent());
550 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
551 } else {
552 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
553 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
554 }
555 }
556 }
557 // PredecessorBlocks will now be empty if for every predecessor of BB we
558 // found a hoistable branch source.
559 return PredecessorBlocks.empty();
560 }
561
562 BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
563 // If BB has already been hoisted, return that
564 if (HoistDestinationMap.count(BB))
565 return HoistDestinationMap[BB];
566
567 // Check if this block is conditional based on a pending branch
568 auto HasBBAsSuccessor =
569 [&](DenseMap::value_type &Pair) {
570 return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
571 Pair.first->getSuccessor(1) == BB);
572 };
573 auto It = std::find_if(HoistableBranches.begin(), HoistableBranches.end(),
574 HasBBAsSuccessor);
575
576 // If not involved in a pending branch, hoist to preheader
577 BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
578 if (It == HoistableBranches.end()) {
579 LLVM_DEBUG(dbgs() << "LICM using " << InitialPreheader->getName()
580 << " as hoist destination for " << BB->getName()
581 << "\n");
582 HoistDestinationMap[BB] = InitialPreheader;
583 return InitialPreheader;
584 }
585 BranchInst *BI = It->first;
586 assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==
587 HoistableBranches.end() &&
588 "BB is expected to be the target of at most one branch");
589
590 LLVMContext &C = BB->getContext();
591 BasicBlock *TrueDest = BI->getSuccessor(0);
592 BasicBlock *FalseDest = BI->getSuccessor(1);
593 BasicBlock *CommonSucc = HoistableBranches[BI];
594 BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());
595
596 // Create hoisted versions of blocks that currently don't have them
597 auto CreateHoistedBlock = [&](BasicBlock *Orig) {
598 if (HoistDestinationMap.count(Orig))
599 return HoistDestinationMap[Orig];
600 BasicBlock *New =
601 BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
602 HoistDestinationMap[Orig] = New;
603 DT->addNewBlock(New, HoistTarget);
604 if (CurLoop->getParentLoop())
605 CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
606 ++NumCreatedBlocks;
607 LLVM_DEBUG(dbgs() << "LICM created " << New->getName()
608 << " as hoist destination for " << Orig->getName()
609 << "\n");
610 return New;
611 };
612 BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
613 BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
614 BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);
615
616 // Link up these blocks with branches.
617 if (!HoistCommonSucc->getTerminator()) {
618 // The new common successor we've generated will branch to whatever that
619 // hoist target branched to.
620 BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
621 assert(TargetSucc && "Expected hoist target to have a single successor");
622 HoistCommonSucc->moveBefore(TargetSucc);
623 BranchInst::Create(TargetSucc, HoistCommonSucc);
624 }
625 if (!HoistTrueDest->getTerminator()) {
626 HoistTrueDest->moveBefore(HoistCommonSucc);
627 BranchInst::Create(HoistCommonSucc, HoistTrueDest);
628 }
629 if (!HoistFalseDest->getTerminator()) {
630 HoistFalseDest->moveBefore(HoistCommonSucc);
631 BranchInst::Create(HoistCommonSucc, HoistFalseDest);
632 }
633
634 // If BI is being cloned to what was originally the preheader then
635 // HoistCommonSucc will now be the new preheader.
636 if (HoistTarget == InitialPreheader) {
637 // Phis in the loop header now need to use the new preheader.
638 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
639 // The new preheader dominates the loop header.
640 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
641 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
642 DT->changeImmediateDominator(HeaderNode, PreheaderNode);
643 // The preheader hoist destination is now the new preheader, with the
644 // exception of the hoist destination of this branch.
645 for (auto &Pair : HoistDestinationMap)
646 if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
647 Pair.second = HoistCommonSucc;
648 }
649
650 // Now finally clone BI.
651 ReplaceInstWithInst(
652 HoistTarget->getTerminator(),
653 BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
654 ++NumClonedBranches;
655
656 assert(CurLoop->getLoopPreheader() &&
657 "Hoisting blocks should not have destroyed preheader");
658 return HoistDestinationMap[BB];
659 }
660 };
661
662439 /// Walk the specified region of the CFG (defined by all blocks dominated by
663440 /// the specified block, and that are in the current loop) in depth first
664441 /// order w.r.t the DominatorTree. This allows us to visit definitions before
673450 CurLoop != nullptr && CurAST != nullptr && SafetyInfo != nullptr &&
674451 "Unexpected input to hoistRegion");
675452
676 ControlFlowHoister CFH(LI, DT, CurLoop);
677
678 // Keep track of instructions that have been hoisted, as they may need to be
679 // re-hoisted if they end up not dominating all of their uses.
680 SmallVector HoistedInstructions;
681
682 // Record what the original preheader is, as we'll need it later if we need to
683 // re-hoist instructions.
684 BasicBlock *OriginalPreheader = CurLoop->getLoopPreheader();
685
686 // For PHI hoisting to work we need to hoist blocks before their successors.
687 // We can do this by iterating through the blocks in the loop in reverse
688 // post-order.
689 LoopBlocksRPO Worklist(CurLoop);
690 Worklist.perform(LI);
453 // We want to visit parents before children. We will enque all the parents
454 // before their children in the worklist and process the worklist in order.
455 SmallVector Worklist = collectChildrenInLoop(N, CurLoop);
456
691457 bool Changed = false;
692 for (BasicBlock *BB : Worklist) {
458 for (DomTreeNode *DTN : Worklist) {
459 BasicBlock *BB = DTN->getBlock();
693460 // Only need to process the contents of this block if it is not part of a
694461 // subloop (which would already have been processed).
695462 if (inSubLoop(BB, CurLoop, LI))
715482 // Try hoisting the instruction out to the preheader. We can only do
716483 // this if all of the operands of the instruction are loop invariant and
717484 // if it is safe to hoist the instruction.
718 // TODO: It may be safe to hoist if we are hoisting to a conditional block
719 // and we have accurately duplicated the control flow from the loop header
720 // to that block.
485 //
721486 if (CurLoop->hasLoopInvariantOperands(&I) &&
722487 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
723488 isSafeToExecuteUnconditionally(
724489 I, DT, CurLoop, SafetyInfo, ORE,
725490 CurLoop->getLoopPreheader()->getTerminator())) {
726 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
727 HoistedInstructions.push_back(&I);
491 hoist(I, DT, CurLoop, SafetyInfo, ORE);
728492 Changed = true;
729493 continue;
730494 }
749513 I.replaceAllUsesWith(Product);
750514 eraseInstruction(I, *SafetyInfo, CurAST);
751515
752 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
753 SafetyInfo, ORE);
754 HoistedInstructions.push_back(ReciprocalDivisor);
516 hoist(*ReciprocalDivisor, DT, CurLoop, SafetyInfo, ORE);
755517 Changed = true;
756518 continue;
757519 }
763525 CurLoop->hasLoopInvariantOperands(&I) &&
764526 SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
765527 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) {
766 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
767 HoistedInstructions.push_back(&I);
528 hoist(I, DT, CurLoop, SafetyInfo, ORE);
768529 Changed = true;
769530 continue;
770531 }
771
772 if (PHINode *PN = dyn_cast(&I)) {
773 if (CFH.canHoistPHI(PN)) {
774 // Redirect incoming blocks first to ensure that we create hoisted
775 // versions of those blocks before we hoist the phi.
776 for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
777 PN->setIncomingBlock(
778 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
779 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
780 ORE);
781 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected");
782 Changed = true;
783 continue;
784 }
785 }
786
787 // Remember possibly hoistable branches so we can actually hoist them
788 // later if needed.
789 if (BranchInst *BI = dyn_cast(&I))
790 CFH.registerPossiblyHoistableBranch(BI);
791 }
792 }
793
794 // If we hoisted instructions to a conditional block they may not dominate
795 // their uses that weren't hoisted (such as phis where some operands are not
796 // loop invariant). If so make them unconditional by moving them to the end of
797 // the original preheader, which is guaranteed to dominate everything in the
798 // loop. We iterate through the instructions in reverse order which ensures
799 // that when we rehoist an instruction we rehoist its operands.
800 Instruction *HoistPoint = OriginalPreheader->getTerminator();
801 for (Instruction *I : reverse(HoistedInstructions)) {
802 if (!llvm::all_of(I->uses(), [&](Use &U) { return DT->dominates(I, U); })) {
803 LLVM_DEBUG(dbgs() << "LICM rehoisting to " << OriginalPreheader->getName()
804 << ": " << *I << "\n");
805 moveInstructionBefore(*I, *HoistPoint, *SafetyInfo);
806 HoistPoint = I;
807 Changed = true;
808 }
809 }
810
811 // Now that we've finished hoisting make sure that LI and DT are still valid.
812 #ifndef NDEBUG
813 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
814 "Dominator tree verification failed");
815 LI->verify(*DT);
816 #endif
532 }
533 }
817534
818535 return Changed;
819536 }
13821099 /// is safe to hoist, this instruction is called to do the dirty work.
13831100 ///
13841101 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1385 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1386 OptimizationRemarkEmitter *ORE) {
1387 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I
1102 ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
1103 auto *Preheader = CurLoop->getLoopPreheader();
1104 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Preheader->getName() << ": " << I
13881105 << "\n");
13891106 ORE->emit([&]() {
13901107 return OptimizationRemark(DEBUG_TYPE, "Hoisted", &I) << "hoisting "
14021119 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
14031120 I.dropUnknownNonDebugMetadata();
14041121
1405 if (isa(I))
1406 // Move the new node to the end of the phi list in the destination block.
1407 moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo);
1408 else
1409 // Move the new node to the destination block, before its terminator.
1410 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo);
1122 // Move the new node to the Preheader, before its terminator.
1123 moveInstructionBefore(I, *Preheader->getTerminator(), *SafetyInfo);
14111124
14121125 // Do not retain debug locations when we are moving instructions to different
14131126 // basic blocks, because we want to avoid jumpy line tables. Calls, however,
+0
-1164
test/Transforms/LICM/hoist-phi.ll less more
None ; RUN: opt -S -licm < %s | FileCheck %s
1 ; RUN: opt -passes='require,loop(licm)' -S < %s | FileCheck %s
2
3 ; CHECK-LABEL: @triangle_phi
4 define void @triangle_phi(i32 %x, i32* %p) {
5 ; CHECK-LABEL: entry:
6 ; CHECK: %cmp1 = icmp sgt i32 %x, 0
7 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
8 entry:
9 br label %loop
10
11 ; CHECK: [[IF_LICM]]:
12 ; CHECK: %add = add i32 %x, 1
13 ; CHECK: br label %[[THEN_LICM]]
14
15 ; CHECK: [[THEN_LICM]]:
16 ; CHECK: phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
17 ; CHECK: store i32 %phi, i32* %p
18 ; CHECK: %cmp2 = icmp ne i32 %phi, 0
19 ; CHECK: br label %loop
20
21 loop:
22 %cmp1 = icmp sgt i32 %x, 0
23 br i1 %cmp1, label %if, label %then
24
25 if:
26 %add = add i32 %x, 1
27 br label %then
28
29 then:
30 %phi = phi i32 [ %add, %if ], [ %x, %loop ]
31 store i32 %phi, i32* %p
32 %cmp2 = icmp ne i32 %phi, 0
33 br i1 %cmp2, label %loop, label %end
34
35 end:
36 ret void
37 }
38
39 ; CHECK-LABEL: @diamond_phi
40 define void @diamond_phi(i32 %x, i32* %p) {
41 ; CHECK-LABEL: entry:
42 ; CHECK: %cmp1 = icmp sgt i32 %x, 0
43 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
44 entry:
45 br label %loop
46
47 ; CHECK: [[IF_LICM]]:
48 ; CHECK: %add = add i32 %x, 1
49 ; CHECK: br label %[[THEN_LICM:.*]]
50
51 ; CHECK: [[ELSE_LICM]]:
52 ; CHECK: %sub = sub i32 %x, 1
53 ; CHECK: br label %[[THEN_LICM]]
54
55 ; CHECK: [[THEN_LICM]]
56 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
57 ; CHECK: store i32 %phi, i32* %p
58 ; CHECK: %cmp2 = icmp ne i32 %phi, 0
59 ; CHECK: br label %loop
60
61 loop:
62 %cmp1 = icmp sgt i32 %x, 0
63 br i1 %cmp1, label %if, label %else
64
65 if:
66 %add = add i32 %x, 1
67 br label %then
68
69 else:
70 %sub = sub i32 %x, 1
71 br label %then
72
73 then:
74 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
75 store i32 %phi, i32* %p
76 %cmp2 = icmp ne i32 %phi, 0
77 br i1 %cmp2, label %loop, label %end
78
79 end:
80 ret void
81 }
82
83 ; TODO: This is currently too complicated for us to be able to hoist the phi.
84 ; CHECK-LABEL: @three_way_phi
85 define void @three_way_phi(i32 %x, i32* %p) {
86 ; CHECK-LABEL: entry:
87 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
88 ; CHECK-DAG: %add = add i32 %x, 1
89 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
90 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
91
92 ; CHECK: [[IF_LICM]]:
93 ; CHECK: br label %[[THEN_LICM:.*]]
94
95 ; CHECK: [[THEN_LICM]]:
96 ; CHECK: %sub = sub i32 %x, 1
97 ; CHECK: br label %loop
98
99 entry:
100 br label %loop
101
102 loop:
103 %cmp1 = icmp sgt i32 %x, 0
104 br i1 %cmp1, label %if, label %then
105
106 if:
107 %add = add i32 %x, 1
108 %cmp2 = icmp sgt i32 %add, 0
109 br i1 %cmp2, label %if.if, label %then
110
111 if.if:
112 %sub = sub i32 %x, 1
113 br label %then
114
115 then:
116 %phi = phi i32 [ 0, %loop ], [ %add, %if ], [ %sub, %if.if ]
117 store i32 %phi, i32* %p
118 %cmp3 = icmp ne i32 %phi, 0
119 br i1 %cmp3, label %loop, label %end
120
121 end:
122 ret void
123 }
124
125 ; TODO: This is currently too complicated for us to be able to hoist the phi.
126 ; CHECK-LABEL: @tree_phi
127 define void @tree_phi(i32 %x, i32* %p) {
128 ; CHECK-LABEL: entry:
129 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
130 ; CHECK-DAG: %add = add i32 %x, 1
131 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
132 ; CHECK-DAG: %sub = sub i32 %x, 1
133 ; CHECK: br label %loop
134
135 entry:
136 br label %loop
137
138 loop:
139 %cmp1 = icmp sgt i32 %x, 0
140 br i1 %cmp1, label %if, label %else
141
142 if:
143 %add = add i32 %x, 1
144 %cmp2 = icmp sgt i32 %add, 0
145 br i1 %cmp2, label %if.if, label %if.else
146
147 if.if:
148 br label %then
149
150 if.else:
151 br label %then
152
153 else:
154 %sub = sub i32 %x, 1
155 br label %then
156
157 then:
158 %phi = phi i32 [ %add, %if.if ], [ 0, %if.else ], [ %sub, %else ]
159 store i32 %phi, i32* %p
160 %cmp3 = icmp ne i32 %phi, 0
161 br i1 %cmp3, label %loop, label %end
162
163 end:
164 ret void
165 }
166
167 ; TODO: We can hoist the first phi, but not the second.
168 ; CHECK-LABEL: @phi_phi
169 define void @phi_phi(i32 %x, i32* %p) {
170 ; CHECK-LABEL: entry:
171 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
172 ; CHECK-DAG: %add = add i32 %x, 1
173 ; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
174 ; CHECK-DAG: %sub = sub i32 %x, 1
175 ; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
176
177 ; CHECK: [[IF_IF_LICM]]:
178 ; CHECK: br label %[[IF_THEN_LICM:.*]]
179
180 ; CHECK: [[IF_ELSE_LICM]]:
181 ; CHECK: br label %[[IF_THEN_LICM]]
182
183 ; CHECK: [[IF_THEN_LICM]]:
184 ; CHECK: %phi1 = phi i32 [ %add, %[[IF_IF_LICM]] ], [ 0, %[[IF_ELSE_LICM]] ]
185 ; CHECK: br label %loop
186
187 entry:
188 br label %loop
189
190 loop:
191 %cmp1 = icmp sgt i32 %x, 0
192 br i1 %cmp1, label %if, label %else
193
194 if:
195 %add = add i32 %x, 1
196 %cmp2 = icmp sgt i32 %add, 0
197 br i1 %cmp2, label %if.if, label %if.else
198
199 if.if:
200 br label %if.then
201
202 if.else:
203 br label %if.then
204
205 if.then:
206 %phi1 = phi i32 [ %add, %if.if ], [ 0, %if.else ]
207 br label %then
208
209 else:
210 %sub = sub i32 %x, 1
211 br label %then
212
213 then:
214 %phi2 = phi i32 [ %phi1, %if.then ], [ %sub, %else ]
215 store i32 %phi2, i32* %p
216 %cmp3 = icmp ne i32 %phi2, 0
217 br i1 %cmp3, label %loop, label %end
218
219 end:
220 ret void
221 }
222
223 ; Check that we correctly duplicate empty control flow.
224 ; CHECK-LABEL: @empty_triangle_phi
225 define i8 @empty_triangle_phi(i32 %x, i32 %y) {
226 ; CHECK-LABEL: entry:
227 ; CHECK: %cmp1 = icmp eq i32 %x, 0
228 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
229 entry:
230 br label %loop
231
232 ; CHECK: [[IF_LICM]]:
233 ; CHECK: br label %[[THEN_LICM]]
234
235 ; CHECK: [[THEN_LICM]]:
236 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
237 ; CHECK: %cmp2 = icmp eq i32 %y, 0
238 ; CHECK: br label %loop
239
240 loop:
241 %cmp1 = icmp eq i32 %x, 0
242 br i1 %cmp1, label %if, label %then
243
244 if:
245 br label %then
246
247 then:
248 %phi = phi i8 [ 0, %if ], [ 1, %loop ]
249 %cmp2 = icmp eq i32 %y, 0
250 br i1 %cmp2, label %end, label %loop
251
252 end:
253 ret i8 %phi
254 }
255
256 ; CHECK-LABEL: @empty_diamond_phi
257 define i8 @empty_diamond_phi(i32 %x, i32 %y) {
258 ; CHECK-LABEL: entry:
259 ; CHECK: %cmp1 = icmp eq i32 %x, 0
260 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
261 entry:
262 br label %loop
263
264 ; CHECK: [[IF_LICM]]:
265 ; CHECK: br label %[[THEN_LICM:.*]]
266
267 ; CHECK: [[ELSE_LICM]]:
268 ; CHECK: br label %[[THEN_LICM]]
269
270 ; CHECK: [[THEN_LICM]]:
271 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
272 ; CHECK: %cmp2 = icmp eq i32 %y, 0
273 ; CHECK: br label %loop
274
275 loop:
276 %cmp1 = icmp eq i32 %x, 0
277 br i1 %cmp1, label %if, label %else
278
279 if:
280 br label %then
281
282 else:
283 br label %then
284
285 then:
286 %phi = phi i8 [ 0, %if ], [ 1, %else ]
287 %cmp2 = icmp eq i32 %y, 0
288 br i1 %cmp2, label %end, label %loop
289
290 end:
291 ret i8 %phi
292 }
293
294 ; Check that we correctly handle the case that the first thing we try to hoist is a phi.
295 ; CHECK-LABEL: @empty_triangle_phi_first
296 define i8 @empty_triangle_phi_first(i32 %x, i1 %cond) {
297 ; CHECK-LABEL: entry:
298 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
299 entry:
300 br label %loop
301
302 ; CHECK: [[IF_LICM]]:
303 ; CHECK: br label %[[THEN_LICM]]
304
305 ; CHECK: [[THEN_LICM]]:
306 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
307 ; CHECK: %cmp = icmp eq i32 %x, 0
308 ; CHECK: br label %loop
309
310 loop:
311 br i1 %cond, label %if, label %then
312
313 if:
314 br label %then
315
316 then:
317 %phi = phi i8 [ 0, %if ], [ 1, %loop ]
318 %cmp = icmp eq i32 %x, 0
319 br i1 %cmp, label %end, label %loop
320
321 end:
322 ret i8 %phi
323 }
324
325 ; CHECK-LABEL: @empty_diamond_phi
326 define i8 @empty_diamond_phi_first(i32 %x, i1 %cond) {
327 ; CHECK-LABEL: entry:
328 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
329 entry:
330 br label %loop
331
332 ; CHECK: [[IF_LICM]]:
333 ; CHECK: br label %[[THEN_LICM:.*]]
334
335 ; CHECK: [[ELSE_LICM]]:
336 ; CHECK: br label %[[THEN_LICM]]
337
338 ; CHECK: [[THEN_LICM]]:
339 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
340 ; CHECK: %cmp = icmp eq i32 %x, 0
341 ; CHECK: br label %loop
342
343 loop:
344 br i1 %cond, label %if, label %else
345
346 if:
347 br label %then
348
349 else:
350 br label %then
351
352 then:
353 %phi = phi i8 [ 0, %if ], [ 1, %else ]
354 %cmp = icmp eq i32 %x, 0
355 br i1 %cmp, label %end, label %loop
356
357 end:
358 ret i8 %phi
359 }
360
361 ; CHECK-LABEL: @empty_triangle_phi_first
362 define i8 @empty_triangle_phi_first_empty_loop_head(i32 %x, i1 %cond) {
363 ; CHECK-LABEL: entry:
364 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
365 entry:
366 br label %loop
367
368 ; CHECK: [[IF_LICM]]:
369 ; CHECK: br label %[[THEN_LICM]]
370
371 ; CHECK: [[THEN_LICM]]:
372 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
373 ; CHECK: %cmp = icmp eq i32 %x, 0
374 ; CHECK: br label %loop
375
376 loop:
377 br label %test
378
379 test:
380 br i1 %cond, label %if, label %then
381
382 if:
383 br label %then
384
385 then:
386 %phi = phi i8 [ 0, %if ], [ 1, %test ]
387 %cmp = icmp eq i32 %x, 0
388 br i1 %cmp, label %end, label %loop
389
390 end:
391 ret i8 %phi
392 }
393
394 ; CHECK-LABEL: @empty_diamond_phi_first_empty_loop_head
395 define i8 @empty_diamond_phi_first_empty_loop_head(i32 %x, i1 %cond) {
396 ; CHECK-LABEL: entry:
397 ; CHECK: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
398 entry:
399 br label %loop
400
401 ; CHECK: [[IF_LICM]]:
402 ; CHECK: br label %[[THEN_LICM:.*]]
403
404 ; CHECK: [[ELSE_LICM]]:
405 ; CHECK: br label %[[THEN_LICM]]
406
407 ; CHECK: [[THEN_LICM]]:
408 ; CHECK: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
409 ; CHECK: %cmp = icmp eq i32 %x, 0
410 ; CHECK: br label %loop
411
412 loop:
413 br label %test
414
415 test:
416 br i1 %cond, label %if, label %else
417
418 if:
419 br label %then
420
421 else:
422 br label %then
423
424 then:
425 %phi = phi i8 [ 0, %if ], [ 1, %else ]
426 %cmp = icmp eq i32 %x, 0
427 br i1 %cmp, label %end, label %loop
428
429 end:
430 ret i8 %phi
431 }
432
433 ; The phi is on one branch of a diamond while simultaneously at the end of a
434 ; triangle. Check that we duplicate the triangle and not the diamond.
435 ; CHECK-LABEL: @triangle_diamond
436 define void @triangle_diamond(i32* %ptr, i32 %x, i32 %y) {
437 ; CHECK-LABEL: entry:
438 ; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
439 ; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
440 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
441 entry:
442 br label %loop
443
444 ; CHECK: [[IF_LICM]]:
445 ; CHECK: br label %[[THEN_LICM]]
446
447 ; CHECK: [[THEN_LICM]]:
448 ; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
449
450 loop:
451 %cmp1 = icmp ne i32 %x, 0
452 br i1 %cmp1, label %if, label %then
453
454 if:
455 %cmp2 = icmp ne i32 %y, 0
456 br i1 %cmp2, label %if.then, label %then
457
458 then:
459 %phi = phi i32 [ 0, %if ], [ 127, %loop ]
460 store i32 %phi, i32* %ptr
461 br label %end
462
463 if.then:
464 br label %end
465
466 end:
467 br label %loop
468 }
469
470 ; As the previous, but the end of the diamond is the head of the loop.
471 ; CHECK-LABEL: @triangle_diamond_backedge
472 define void @triangle_diamond_backedge(i32* %ptr, i32 %x, i32 %y) {
473 ; CHECK-LABEL: entry:
474 ; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
475 ; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
476 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
477 entry:
478 br label %loop
479
480 ; CHECK: [[IF_LICM]]:
481 ; CHECK: br label %[[THEN_LICM]]
482
483 ; CHECK: [[THEN_LICM]]:
484 ; CHECK: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
485
486 loop:
487 %cmp1 = icmp ne i32 %x, 0
488 br i1 %cmp1, label %if, label %then
489
490 if:
491 %cmp2 = icmp ne i32 %y, 0
492 br i1 %cmp2, label %backedge, label %then
493
494 then:
495 %phi = phi i32 [ 0, %if ], [ 127, %loop ]
496 store i32 %phi, i32* %ptr
497 br label %loop
498
499 backedge:
500 br label %loop
501 }
502
503 ; TODO: The inner diamonds can be hoisted, but not currently the outer diamond
504 ; CHECK-LABEL: @diamonds_inside_diamond
505 define void @diamonds_inside_diamond(i32 %x, i32* %p) {
506 ; CHECK-LABEL: entry:
507 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
508 ; CHECK-DAG: %cmp3 = icmp slt i32 %x, -10
509 ; CHECK: br i1 %cmp3, label %[[ELSE_IF_LICM:.*]], label %[[ELSE_ELSE_LICM:.*]]
510 entry:
511 br label %loop
512
513 ; CHECK: [[ELSE_IF_LICM]]:
514 ; CHECK: br label %[[ELSE_THEN_LICM:.*]]
515
516 ; CHECK: [[ELSE_ELSE_LICM]]:
517 ; CHECK: br label %[[ELSE_THEN_LICM]]
518
519 ; CHECK: [[ELSE_THEN_LICM]]:
520 ; CHECK: %phi2 = phi i32 [ 2, %[[ELSE_IF_LICM]] ], [ 3, %[[ELSE_ELSE_LICM]] ]
521 ; CHECK: %cmp2 = icmp sgt i32 %x, 10
522 ; CHECK: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
523
524 ; CHECK: [[IF_IF_LICM]]:
525 ; CHECK: br label %[[IF_THEN_LICM:.*]]
526
527 ; CHECK: [[IF_ELSE_LICM]]:
528 ; CHECK: br label %[[IF_THEN_LICM]]
529
530 ; CHECK: [[IF_THEN_LICM]]:
531 ; CHECK: %phi1 = phi i32 [ 0, %[[IF_IF_LICM]] ], [ 1, %[[IF_ELSE_LICM]] ]
532 ; CHECK: br label %loop
533
534 loop:
535 %cmp1 = icmp sgt i32 %x, 0
536 br i1 %cmp1, label %if, label %else
537
538 if:
539 %cmp2 = icmp sgt i32 %x, 10
540 br i1 %cmp2, label %if.if, label %if.else
541
542 if.if:
543 br label %if.then
544
545 if.else:
546 br label %if.then
547
548 if.then:
549 %phi1 = phi i32 [ 0, %if.if ], [ 1, %if.else ]
550 br label %then
551
552 else:
553 %cmp3 = icmp slt i32 %x, -10
554 br i1 %cmp3, label %else.if, label %else.else
555
556 else.if:
557 br label %else.then
558
559 else.else:
560 br label %else.then
561
562 else.then:
563 %phi2 = phi i32 [ 2, %else.if ], [ 3, %else.else ]
564 br label %then
565
566 then:
567 %phi3 = phi i32 [ %phi1, %if.then ], [ %phi2, %else.then ]
568 store i32 %phi3, i32* %p
569 %cmp4 = icmp ne i32 %phi3, 0
570 br i1 %cmp4, label %loop, label %end
571
572 end:
573 ret void
574 }
575
576 ; We can hoist blocks that contain an edge that exits the loop by ignoring that
577 ; edge in the hoisted block.
578 ; CHECK-LABEL: @triangle_phi_loopexit
579 define void @triangle_phi_loopexit(i32 %x, i32* %p) {
580 ; CHECK-LABEL: entry:
581 ; CHECK-DAG: %add = add i32 %x, 1
582 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
583 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
584 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
585 entry:
586 br label %loop
587
588 ; CHECK: [[IF_LICM]]:
589 ; CHECK: br label %[[THEN_LICM]]
590
591 ; CHECK: [[THEN_LICM]]:
592 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
593
594 loop:
595 %cmp1 = icmp sgt i32 %x, 0
596 br i1 %cmp1, label %if, label %then
597
598 if:
599 %add = add i32 %x, 1
600 %cmp2 = icmp sgt i32 10, %add
601 br i1 %cmp2, label %then, label %end
602
603 then:
604 %phi = phi i32 [ %add, %if ], [ %x, %loop ]
605 store i32 %phi, i32* %p
606 %cmp3 = icmp ne i32 %phi, 0
607 br i1 %cmp3, label %loop, label %end
608
609 end:
610 ret void
611 }
612
613 ; CHECK-LABEL: @diamond_phi_oneloopexit
614 define void @diamond_phi_oneloopexit(i32 %x, i32* %p) {
615 ; CHECK-LABEL: entry:
616 ; CHECK-DAG: %add = add i32 %x, 1
617 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
618 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
619 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
620 entry:
621 br label %loop
622
623 ; CHECK: [[IF_LICM]]:
624 ; CHECK: br label %[[THEN_LICM:.*]]
625
626 ; CHECK: [[ELSE_LICM]]:
627 ; CHECK: %sub = sub i32 %x, 1
628 ; CHECK: br label %[[THEN_LICM]]
629
630 ; CHECK: [[THEN_LICM]]
631 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
632 ; CHECK: %cmp3 = icmp ne i32 %phi, 0
633 ; CHECK: br label %loop
634
635 loop:
636 %cmp1 = icmp sgt i32 %x, 0
637 br i1 %cmp1, label %if, label %else
638
639 if:
640 %add = add i32 %x, 1
641 %cmp2 = icmp sgt i32 10, %add
642 br i1 %cmp2, label %then, label %end
643
644 else:
645 %sub = sub i32 %x, 1
646 br label %then
647
648 then:
649 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
650 store i32 %phi, i32* %p
651 %cmp3 = icmp ne i32 %phi, 0
652 br i1 %cmp3, label %loop, label %end
653
654 end:
655 ret void
656 }
657
658 ; CHECK-LABEL: @diamond_phi_twoloopexit
659 define void @diamond_phi_twoloopexit(i32 %x, i32* %p) {
660 ; CHECK-LABEL: entry:
661 ; CHECK-DAG: %sub = sub i32 %x, 1
662 ; CHECK-DAG: %add = add i32 %x, 1
663 ; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
664 ; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
665 ; CHECK-DAG: %cmp3 = icmp sgt i32 10, %sub
666 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
667 entry:
668 br label %loop
669
670 ; CHECK: [[IF_LICM]]:
671 ; CHECK: br label %[[THEN_LICM:.*]]
672
673 ; CHECK: [[ELSE_LICM]]:
674 ; CHECK: br label %[[THEN_LICM]]
675
676 ; CHECK: [[THEN_LICM]]
677 ; CHECK: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
678 ; CHECK: %cmp4 = icmp ne i32 %phi, 0
679 ; CHECK: br label %loop
680
681 loop:
682 %cmp1 = icmp sgt i32 %x, 0
683 br i1 %cmp1, label %if, label %else
684
685 if:
686 %add = add i32 %x, 1
687 %cmp2 = icmp sgt i32 10, %add
688 br i1 %cmp2, label %then, label %end
689
690 else:
691 %sub = sub i32 %x, 1
692 %cmp3 = icmp sgt i32 10, %sub
693 br i1 %cmp3, label %then, label %end
694
695 then:
696 %phi = phi i32 [ %add, %if ], [ %sub, %else ]
697 store i32 %phi, i32* %p
698 %cmp4 = icmp ne i32 %phi, 0
699 br i1 %cmp4, label %loop, label %end
700
701 end:
702 ret void
703 }
704
705 ; The store cannot be hoisted, so add and shr cannot be hoisted into a
706 ; conditional block.
707 ; CHECK-LABEL: @conditional_use
708 define void @conditional_use(i32 %x, i32* %p) {
709 ; CHECK-LABEL: entry:
710 ; CHECK-DAG: %cond = icmp ugt i32 %x, 0
711 ; CHECK-DAG: %add = add i32 %x, 5
712 ; CHECK-DAG: %shr = ashr i32 %add, 1
713 ; CHECK: br label %loop
714 entry:
715 br label %loop
716
717 loop:
718 %cond = icmp ugt i32 %x, 0
719 br i1 %cond, label %if, label %else
720
721 ; CHECK-LABEL: if:
722 ; CHECK: store i32 %shr, i32* %p, align 4
723 if:
724 %add = add i32 %x, 5
725 %shr = ashr i32 %add, 1
726 store i32 %shr, i32* %p, align 4
727 br label %then
728
729 else:
730 br label %then
731
732 then:
733 br label %loop
734 }
735
736 ; A diamond with two triangles on the left and one on the right. This test is
737 ; to check that we have a unique loop preheader when we hoist the store (and so
738 ; don't fail an assertion).
739 ; CHECK-LABEL: @triangles_in_diamond
740 define void @triangles_in_diamond(i32* %ptr) {
741 ; CHECK-LABEL: entry:
742 ; CHECK: store i32 0, i32* %ptr, align 4
743 ; CHECK: br label %loop
744 entry:
745 br label %loop
746
747 loop:
748 br i1 undef, label %left_triangle_1, label %right_triangle
749
750 left_triangle_1:
751 br i1 undef, label %left_triangle_1_if, label %left_triangle_2
752
753 left_triangle_1_if:
754 br label %left_triangle_2
755
756 left_triangle_2:
757 br i1 undef, label %left_triangle_2_if, label %left_triangle_2_then
758
759 left_triangle_2_if:
760 br label %left_triangle_2_then
761
762 left_triangle_2_then:
763 br label %loop.end
764
765 right_triangle:
766 br i1 undef, label %right_triangle.if, label %right_triangle.then
767
768 right_triangle.if:
769 br label %right_triangle.then
770
771 right_triangle.then:
772 br label %loop.end
773
774 loop.end:
775 store i32 0, i32* %ptr, align 4
776 br label %loop
777 }
778
779 ; %cmp dominates its used after being hoisted, but not after %brmerge is rehoisted
780 ; CHECK-LABEL: @rehoist
781 define void @rehoist(i8* %this, i32 %x) {
782 ; CHECK-LABEL: entry:
783 ; CHECK-DAG: %sub = add nsw i32 %x, -1
784 ; CHECK-DAG: %fptr = bitcast i8* %this to void (i8*)*
785 ; CHECK-DAG: %cmp = icmp eq i32 0, %sub
786 ; CHECK-DAG: %brmerge = or i1 %cmp, true
787 entry:
788 %sub = add nsw i32 %x, -1
789 br label %loop
790
791 loop:
792 br i1 undef, label %if1, label %else1
793
794 if1:
795 %fptr = bitcast i8* %this to void (i8*)*
796 call void %fptr(i8* %this)
797 br label %then1
798
799 else1:
800 br label %then1
801
802 then1:
803 %cmp = icmp eq i32 0, %sub
804 br i1 %cmp, label %end, label %else2
805
806 else2:
807 %brmerge = or i1 %cmp, true
808 br i1 %brmerge, label %if3, label %end
809
810 if3:
811 br label %end
812
813 end:
814 br label %loop
815 }
816
817 ; A test case that uses empty blocks in a way that can cause control flow
818 ; hoisting to get confused.
819 ; CHECK-LABEL: @empty_blocks_multiple_conditional_branches
820 define void @empty_blocks_multiple_conditional_branches(float %arg, float* %ptr) {
821 ; CHECK-LABEL: entry
822 ; CHECK-DAG: %div1 = fmul float %arg, 4.000000e+00
823 ; CHECK-DAG: %div2 = fmul float %arg, 2.000000e+00
824 entry:
825 br label %loop
826
827 ; The exact path to the phi isn't checked here, because it depends on whether
828 ; cond2 or cond3 is hoisted first
829 ; CHECK: %phi = phi float [ 0.000000e+00, %{{.*}} ], [ %div1, %{{.*}} ]
830 ; CHECK: br label %loop
831
832 loop:
833 br i1 undef, label %backedge2, label %cond1
834
835 cond1:
836 br i1 undef, label %cond1.if, label %cond1.else
837
838 cond1.else:
839 br label %cond3
840
841 cond1.if:
842 br label %cond1.if.next
843
844 cond1.if.next:
845 br label %cond2
846
847 cond2:
848 %div1 = fmul float %arg, 4.000000e+00
849 br i1 undef, label %cond2.if, label %cond2.then
850
851 cond2.if:
852 br label %cond2.then
853
854 cond2.then:
855 %phi = phi float [ 0.000000e+00, %cond2 ], [ %div1, %cond2.if ]
856 store float %phi, float* %ptr
857 br label %backedge2
858
859 cond3:
860 br i1 undef, label %cond3.then, label %cond3.if
861
862 cond3.if:
863 %div2 = fmul float %arg, 2.000000e+00
864 store float %div2, float* %ptr
865 br label %cond3.then
866
867 cond3.then:
868 br label %loop
869
870 backedge2:
871 br label %loop
872 }
873
874 ; We can't do much here, so mainly just check that we don't crash.
875 ; CHECK-LABEL: @many_path_phi
876 define void @many_path_phi(i32* %ptr1, i32* %ptr2) {
877 ; CHECK-LABEL: entry:
878 ; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
879 ; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
880 ; CHECK: br label %loop
881 entry:
882 br label %loop
883
884 loop:
885 %phi1 = phi i32 [ 0, %entry ], [ %phi2, %end ]
886 %cmp1 = icmp ugt i32 %phi1, 3
887 br i1 %cmp1, label %cond2, label %cond1
888
889 cond1:
890 br i1 undef, label %end, label %cond1.else
891
892 cond1.else:
893 %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
894 %val2 = load i32, i32* %gep2, align 4
895 %cmp2 = icmp eq i32 %val2, 13
896 br i1 %cmp2, label %cond1.end, label %end
897
898 cond1.end:
899 br label %end
900
901 cond2:
902 br i1 undef, label %end, label %cond2.else
903
904 cond2.else:
905 %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
906 %val3 = load i32, i32* %gep3, align 4
907 %cmp3 = icmp eq i32 %val3, 13
908 br i1 %cmp3, label %cond2.end, label %end
909
910 cond2.end:
911 br label %end
912
913 end:
914 %phi2 = phi i32 [ 1, %cond1 ], [ 2, %cond1.else ], [ 3, %cond1.end ], [ 4, %cond2 ], [ 5, %cond2.else ], [ 6, %cond2.end ]
915 br label %loop
916 }
917
918 ; Check that we correctly handle the hoisting of %gep when theres a critical
919 ; edge that branches to the preheader.
920 ; CHECK-LABEL: @crit_edge
921 define void @crit_edge(i32* %ptr, i32 %idx, i1 %cond1, i1 %cond2) {
922 ; CHECK-LABEL: entry:
923 ; CHECK: %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
924 ; CHECK: br label %preheader
925 entry:
926 br label %preheader
927
928 preheader:
929 br label %loop
930
931 loop:
932 br i1 %cond1, label %then, label %if
933
934 if:
935 %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
936 %val = load i32, i32* %gep
937 br label %then
938
939 then:
940 %phi = phi i32 [ %val, %if ], [ 0, %loop ]
941 store i32 %phi, i32* %ptr
942 br i1 %cond2, label %loop, label %crit_edge
943
944 crit_edge:
945 br label %preheader
946 }
947
948 ; Check that the conditional sub is correctly hoisted from the inner loop to the
949 ; preheader of the outer loop.
950 ; CHECK-LABEL: @hoist_from_innermost_loop
951 define void @hoist_from_innermost_loop(i32 %nx, i32* %ptr) {
952 ; CHECK-LABEL: entry:
953 ; CHECK-DAG: %sub = sub nsw i32 0, %nx
954 ; CHECK: br label %outer_loop
955 entry:
956 br label %outer_loop
957
958 outer_loop:
959 br label %middle_loop
960
961 middle_loop:
962 br label %inner_loop
963
964 inner_loop:
965 br i1 undef, label %inner_loop_end, label %if
966
967 if:
968 %sub = sub nsw i32 0, %nx
969 store i32 %sub, i32* %ptr, align 4
970 br label %inner_loop_end
971
972 inner_loop_end:
973 br i1 undef, label %inner_loop, label %middle_loop_end
974
975 middle_loop_end:
976 br i1 undef, label %middle_loop, label %outer_loop_end
977
978 outer_loop_end:
979 br label %outer_loop
980 }
981
982 ; We have a diamond starting from %if, but %if.if is also reachable from %loop,
983 ; so %gep should not be conditionally hoisted.
984 ; CHECK-LABEL: @diamond_with_extra_in_edge
985 define void @diamond_with_extra_in_edge(i32* %ptr1, i32* %ptr2, i32 %arg) {
986 ; CHECK-LABEL: entry:
987 ; CHECK-DAG: %cmp2 = icmp ne i32 0, %arg
988 ; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
989 ; CHECK: br label %loop
990 entry:
991 br label %loop
992
993 loop:
994 %phi1 = phi i32 [ 0, %entry ], [ %phi2, %then ]
995 %cmp1 = icmp ugt i32 16, %phi1
996 br i1 %cmp1, label %if, label %if.if
997
998 if:
999 %cmp2 = icmp ne i32 0, %arg
1000 br i1 %cmp2, label %if.if, label %if.else
1001
1002 if.if:
1003 %gep = getelementptr i32, i32* %ptr1, i32 4
1004 %val = load i32, i32* %gep, align 4
1005 br label %then
1006
1007 if.else:
1008 br label %then
1009
1010 then:
1011 %phi2 = phi i32 [ %val, %if.if ], [ %phi1, %if.else ]
1012 store i32 %phi2, i32* %ptr2, align 4
1013 br label %loop
1014 }
1015
1016 ; %loop/%if/%then form a triangle, but %loop/%if/%then/%end also form a diamond.
1017 ; The triangle should be picked for conditional hoisting.
1018 ; CHECK-LABEL: @both_triangle_and_diamond
1019 define void @both_triangle_and_diamond(i32* %ptr1, i32* %ptr2, i32 %arg) {
1020 ; CHECK-LABEL: entry:
1021 ; CHECK-DAG: %cmp1 = icmp ne i32 0, %arg
1022 ; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
1023 ; CHECK: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
1024 entry:
1025 br label %loop
1026
1027 ; CHECK: [[IF_LICM]]:
1028 ; CHECK: br label %[[THEN_LICM]]
1029
1030 ; CHECK: [[THEN_LICM]]:
1031 ; CHECK: %phi2 = phi i32 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
1032 ; CHECK: br label %loop
1033
1034 loop:
1035 %phi1 = phi i32 [ 0, %entry ], [ %phi3, %end ]
1036 %cmp1 = icmp ne i32 0, %arg
1037 br i1 %cmp1, label %if, label %then
1038
1039 if:
1040 %gep = getelementptr i32, i32* %ptr1, i32 4
1041 %val = load i32, i32* %gep, align 4
1042 %cmp2 = icmp ugt i32 16, %phi1
1043 br i1 %cmp2, label %end, label %then
1044
1045 then:
1046 %phi2 = phi i32 [ 0, %if ], [ 1, %loop ]
1047 br label %end
1048
1049 end:
1050 %phi3 = phi i32 [ %phi2, %then ], [ %val, %if ]
1051 store i32 %phi3, i32* %ptr2, align 4
1052 br label %loop
1053 }
1054
1055 ; We shouldn't duplicate the branch at the end of %loop and should instead hoist
1056 ; %val to %entry.
1057 ; CHECK-LABEL: @same_destination_branch
1058 define i32 @same_destination_branch(i32 %arg1, i32 %arg2) {
1059 ; CHECK-LABEL: entry:
1060 ; CHECK-DAG: %cmp1 = icmp ne i32 %arg2, 0
1061 ; CHECK-DAG: %val = add i32 %arg1, 1
1062 ; CHECK: br label %loop
1063 entry:
1064 br label %loop
1065
1066 loop:
1067 %phi = phi i32 [ 0, %entry ], [ %add, %then ]
1068 %add = add i32 %phi, 1
1069 %cmp1 = icmp ne i32 %arg2, 0
1070 br i1 %cmp1, label %if, label %if
1071
1072 if:
1073 %val = add i32 %arg1, 1
1074 br label %then
1075
1076 then:
1077 %cmp2 = icmp ne i32 %val, %phi
1078 br i1 %cmp2, label %loop, label %end
1079
1080 end:
1081 ret i32 %val
1082 }
1083
1084 ; Diamond-like control flow but the left/right blocks actually have the same
1085 ; destinations.
1086 ; TODO: We could potentially hoist all of phi2-4, but currently only hoist phi2.
1087 ; CHECK-LABEL: @diamond_like_same_destinations
1088 define i32 @diamond_like_same_destinations(i32 %arg1, i32 %arg2) {
1089 ; CHECK-LABEL: entry:
1090 ; CHECK-DAG: %cmp1 = icmp ne i32 %arg1, 0
1091 ; CHECK-DAG: %cmp2 = icmp ugt i32 %arg2, 1
1092 ; CHECK-DAG: %cmp3 = icmp ugt i32 %arg2, 2
1093 ; CHECK: br i1 %cmp1, label %[[LEFT1_LICM:.*]], label %[[RIGHT1_LICM:.*]]
1094 entry:
1095 br label %loop
1096
1097 ; CHECK: [[LEFT1_LICM]]:
1098 ; CHECK: br label %[[LEFT2_LICM:.*]]
1099
1100 ; CHECK: [[RIGHT1_LICM]]:
1101 ; CHECK: br label %[[LEFT2_LICM]]
1102
1103 ; CHECK: [[LEFT2_LICM]]:
1104 ; CHECK: %phi2 = phi i32 [ 0, %[[LEFT1_LICM]] ], [ 1, %[[RIGHT1_LICM]] ]
1105 ; CHECK: br label %loop
1106
1107 loop:
1108 %phi1 = phi i32 [ 0, %entry ], [ %add, %loopend ]
1109 %add = add i32 %phi1, 1
1110 %cmp1 = icmp ne i32 %arg1, 0
1111 br i1 %cmp1, label %left1, label %right1
1112
1113 left1:
1114 %cmp2 = icmp ugt i32 %arg2, 1
1115 br i1 %cmp2, label %left2, label %right2
1116
1117 right1:
1118 %cmp3 = icmp ugt i32 %arg2, 2
1119 br i1 %cmp3, label %left2, label %right2
1120
1121 left2:
1122 %phi2 = phi i32 [ 0, %left1 ], [ 1, %right1 ]
1123 br label %loopend
1124
1125 right2:
1126 %phi3 = phi i32 [ 2, %left1 ], [ 3, %right1 ]
1127 br label %loopend
1128
1129 loopend:
1130 %phi4 = phi i32 [ %phi2, %left2 ], [ %phi3, %right2 ]
1131 %cmp4 = icmp ne i32 %phi1, 32
1132 br i1 %cmp4, label %loop, label %end
1133
1134 end:
1135 ret i32 %phi4
1136 }
1137
1138 ; A phi with multiple incoming values for the same block due to a branch with
1139 ; two destinations that are actually the same. We can't hoist this.
1140 ; TODO: This could be hoisted by erasing one of the incoming values.
1141 ; CHECK-LABEL: @phi_multiple_values_same_block
1142 define i32 @phi_multiple_values_same_block(i32 %arg) {
1143 ; CHECK-LABEL: entry:
1144 ; CHECK: %cmp = icmp sgt i32 %arg, 4
1145 ; CHECK-NOT: phi
1146 ; CHECK: br label %loop
1147 entry:
1148 br label %loop
1149
1150 loop:
1151 %cmp = icmp sgt i32 %arg, 4
1152 br i1 %cmp, label %if, label %then
1153
1154 if:
1155 br i1 undef, label %then, label %then
1156
1157 then:
1158 %phi = phi i32 [ %arg, %loop ], [ 1, %if ], [ 1, %if ]
1159 br i1 undef, label %exit, label %loop
1160
1161 exit:
1162 ret i32 %phi
1163 }
265265 ; variant/invariant values being stored to invariant address.
266266 ; test checks that the last element of the phi is extracted and scalar stored
267267 ; into the uniform address within the loop.
268 ; Since the condition and the phi is loop invariant, they are LICM'ed before
268 ; Since the condition and the phi is loop invariant, they are LICM'ed after
269269 ; vectorization.
270270 ; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv
271271 ; CHECK-NEXT: entry:
272 ; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
273 ; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
274272 ; CHECK-NEXT: [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
275273 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]]
276 ; CHECK-NEXT: br i1 [[CMP]], label %[[COND_STORE_LICM:.*]], label %[[COND_STORE_K_LICM:.*]]
277 ; CHECK: [[COND_STORE_LICM]]:
278 ; CHECK-NEXT: br label %[[LATCH_LICM:.*]]
279 ; CHECK: [[COND_STORE_K_LICM]]:
280 ; CHECK-NEXT: br label %[[LATCH_LICM]]
281 ; CHECK: [[LATCH_LICM]]:
282 ; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], %[[COND_STORE_LICM]] ], [ [[K]], %[[COND_STORE_K_LICM]] ]
283274 ; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
284275 ; CHECK-NEXT: [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
285276 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
286277 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
287278 ; CHECK: vector.memcheck:
279 ; CHECK-NEXT: [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
280 ; CHECK-NEXT: [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
288281 ; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
289282 ; CHECK-NEXT: [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
290283 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
297290 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
298291 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[NTRUNC]], i32 0
299292 ; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer
293 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i32 3
294 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[K]], i32 3
295 ; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[TMP3]]
296 ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
300297 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
301298 ; CHECK: vector.body:
302299 ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
303300 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
304301 ; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
305302 ; CHECK-NEXT: store <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32>* [[TMP7]], align 4
306 ; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4
303 ; CHECK-NEXT: store i32 [[TMP5]], i32* [[A]], align 4
307304 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
308305 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
309306 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
323320 ; CHECK: cond_store_k:
324321 ; CHECK-NEXT: br label [[LATCH]]
325322 ; CHECK: latch:
323 ; CHECK-NEXT: [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ]
326324 ; CHECK-NEXT: store i32 [[STOREVAL]], i32* [[A]], align 4
327325 ; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
328326 ; CHECK-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]