llvm.org GIT mirror llvm / 9a60319
Update MemorySSA in SimpleLoopUnswitch. Summary: Teach SimpleLoopUnswitch to preserve MemorySSA. Subscribers: sanjoy, jlebar, Prazek, george.burgess.iv, llvm-commits Differential Revision: https://reviews.llvm.org/D47022 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@348263 91177308-0d34-0410-b5e6-96231b3b80d8 Alina Sbirlea 9 months ago
32 changed file(s) with 267 addition(s) and 78 deletion(s). Raw diff Collapse all Expand all
2424 #include "llvm/Analysis/LoopInfo.h"
2525 #include "llvm/Analysis/LoopIterator.h"
2626 #include "llvm/Analysis/LoopPass.h"
27 #include "llvm/Analysis/MemorySSA.h"
28 #include "llvm/Analysis/MemorySSAUpdater.h"
2729 #include "llvm/Analysis/Utils/Local.h"
2830 #include "llvm/IR/BasicBlock.h"
2931 #include "llvm/IR/Constant.h"
348350 /// If `SE` is not null, it will be updated based on the potential loop SCEVs
349351 /// invalidated by this.
350352 static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
351 LoopInfo &LI, ScalarEvolution *SE) {
353 LoopInfo &LI, ScalarEvolution *SE,
354 MemorySSAUpdater *MSSAU) {
352355 assert(BI.isConditional() && "Can only unswitch a conditional branch!");
353356 LLVM_DEBUG(dbgs() << " Trying to unswitch branch: " << BI << "\n");
354357
422425 SE->forgetTopmostLoop(&L);
423426 }
424427
428 if (MSSAU && VerifyMemorySSA)
429 MSSAU->getMemorySSA()->verifyMemorySSA();
430
425431 // Split the preheader, so that we know that there is a safe place to insert
426432 // the conditional branch. We will change the preheader to have a conditional
427433 // branch on LoopCond.
428434 BasicBlock *OldPH = L.getLoopPreheader();
429 BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI);
435 BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI, MSSAU);
430436
431437 // Now that we have a place to insert the conditional branch, create a place
432438 // to branch to: this is the exit block out of the loop that we are
438444 "A branch's parent isn't a predecessor!");
439445 UnswitchedBB = LoopExitBB;
440446 } else {
441 UnswitchedBB = SplitBlock(LoopExitBB, &LoopExitBB->front(), &DT, &LI);
442 }
447 UnswitchedBB =
448 SplitBlock(LoopExitBB, &LoopExitBB->front(), &DT, &LI, MSSAU);
449 }
450
451 if (MSSAU && VerifyMemorySSA)
452 MSSAU->getMemorySSA()->verifyMemorySSA();
443453
444454 // Actually move the invariant uses into the unswitched position. If possible,
445455 // we do this by moving the instructions, but when doing partial unswitching
451461 // its successors.
452462 OldPH->getInstList().splice(OldPH->end(), BI.getParent()->getInstList(),
453463 BI);
464 if (MSSAU) {
465 // Temporarily clone the terminator, to make MSSA update cheaper by
466 // separating "insert edge" updates from "remove edge" ones.
467 ParentBB->getInstList().push_back(BI.clone());
468 } else {
469 // Create a new unconditional branch that will continue the loop as a new
470 // terminator.
471 BranchInst::Create(ContinueBB, ParentBB);
472 }
454473 BI.setSuccessor(LoopExitSuccIdx, UnswitchedBB);
455474 BI.setSuccessor(1 - LoopExitSuccIdx, NewPH);
456
457 // Create a new unconditional branch that will continue the loop as a new
458 // terminator.
459 BranchInst::Create(ContinueBB, ParentBB);
460475 } else {
461476 // Only unswitching a subset of inputs to the condition, so we will need to
462477 // build a new branch that merges the invariant inputs.
472487 *UnswitchedBB, *NewPH);
473488 }
474489
490 // Update the dominator tree with the added edge.
491 DT.insertEdge(OldPH, UnswitchedBB);
492
493 // After the dominator tree was updated with the added edge, update MemorySSA
494 // if available.
495 if (MSSAU) {
496 SmallVector Updates;
497 Updates.push_back({cfg::UpdateKind::Insert, OldPH, UnswitchedBB});
498 MSSAU->applyInsertUpdates(Updates, DT);
499 }
500
501 // Finish updating dominator tree and memory ssa for full unswitch.
502 if (FullUnswitch) {
503 if (MSSAU) {
504 // Remove the cloned branch instruction.
505 ParentBB->getTerminator()->eraseFromParent();
506 // Create unconditional branch now.
507 BranchInst::Create(ContinueBB, ParentBB);
508 MSSAU->removeEdge(ParentBB, LoopExitBB);
509 }
510 DT.deleteEdge(ParentBB, LoopExitBB);
511 }
512
513 if (MSSAU && VerifyMemorySSA)
514 MSSAU->getMemorySSA()->verifyMemorySSA();
515
475516 // Rewrite the relevant PHI nodes.
476517 if (UnswitchedBB == LoopExitBB)
477518 rewritePHINodesForUnswitchedExitBlock(*UnswitchedBB, *ParentBB, *OldPH);
478519 else
479520 rewritePHINodesForExitAndUnswitchedBlocks(*LoopExitBB, *UnswitchedBB,
480521 *ParentBB, *OldPH, FullUnswitch);
481
482 // Now we need to update the dominator tree.
483 SmallVector DTUpdates;
484 DTUpdates.push_back({DT.Insert, OldPH, UnswitchedBB});
485 if (FullUnswitch)
486 DTUpdates.push_back({DT.Delete, ParentBB, LoopExitBB});
487 DT.applyUpdates(DTUpdates);
488522
489523 // The constant we can replace all of our invariants with inside the loop
490524 // body. If any of the invariants have a value other than this the loop won't
536570 /// If `SE` is not null, it will be updated based on the potential loop SCEVs
537571 /// invalidated by this.
538572 static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT,
539 LoopInfo &LI, ScalarEvolution *SE) {
573 LoopInfo &LI, ScalarEvolution *SE,
574 MemorySSAUpdater *MSSAU) {
540575 LLVM_DEBUG(dbgs() << " Trying to unswitch switch: " << SI << "\n");
541576 Value *LoopCond = SI.getCondition();
542577
562597 return false;
563598
564599 LLVM_DEBUG(dbgs() << " unswitching trivial switch...\n");
600
601 if (MSSAU && VerifyMemorySSA)
602 MSSAU->getMemorySSA()->verifyMemorySSA();
565603
566604 // We may need to invalidate SCEVs for the outermost loop reached by any of
567605 // the exits.
625663 // Split the preheader, so that we know that there is a safe place to insert
626664 // the switch.
627665 BasicBlock *OldPH = L.getLoopPreheader();
628 BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI);
666 BasicBlock *NewPH = SplitEdge(OldPH, L.getHeader(), &DT, &LI, MSSAU);
629667 OldPH->getTerminator()->eraseFromParent();
630668
631669 // Now add the unswitched switch.
648686 rewritePHINodesForUnswitchedExitBlock(*DefaultExitBB, *ParentBB, *OldPH);
649687 } else {
650688 auto *SplitBB =
651 SplitBlock(DefaultExitBB, &DefaultExitBB->front(), &DT, &LI);
652 rewritePHINodesForExitAndUnswitchedBlocks(
653 *DefaultExitBB, *SplitBB, *ParentBB, *OldPH, /*FullUnswitch*/ true);
689 SplitBlock(DefaultExitBB, &DefaultExitBB->front(), &DT, &LI, MSSAU);
690 rewritePHINodesForExitAndUnswitchedBlocks(*DefaultExitBB, *SplitBB,
691 *ParentBB, *OldPH,
692 /*FullUnswitch*/ true);
654693 DefaultExitBB = SplitExitBBMap[DefaultExitBB] = SplitBB;
655694 }
656695 }
674713 BasicBlock *&SplitExitBB = SplitExitBBMap[ExitBB];
675714 if (!SplitExitBB) {
676715 // If this is the first time we see this, do the split and remember it.
677 SplitExitBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI);
678 rewritePHINodesForExitAndUnswitchedBlocks(
679 *ExitBB, *SplitExitBB, *ParentBB, *OldPH, /*FullUnswitch*/ true);
716 SplitExitBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI, MSSAU);
717 rewritePHINodesForExitAndUnswitchedBlocks(*ExitBB, *SplitExitBB,
718 *ParentBB, *OldPH,
719 /*FullUnswitch*/ true);
680720 }
681721 // Update the case pair to point to the split block.
682722 CasePair.second = SplitExitBB;
753793 DTUpdates.push_back({DT.Insert, OldPH, UnswitchedBB});
754794 }
755795 DT.applyUpdates(DTUpdates);
796
797 if (MSSAU) {
798 MSSAU->applyUpdates(DTUpdates, DT);
799 if (VerifyMemorySSA)
800 MSSAU->getMemorySSA()->verifyMemorySSA();
801 }
802
756803 assert(DT.verify(DominatorTree::VerificationLevel::Fast));
757804
758805 // We may have changed the nesting relationship for this loop so hoist it to
778825 /// If `SE` is not null, it will be updated based on the potential loop SCEVs
779826 /// invalidated by this.
780827 static bool unswitchAllTrivialConditions(Loop &L, DominatorTree &DT,
781 LoopInfo &LI, ScalarEvolution *SE) {
828 LoopInfo &LI, ScalarEvolution *SE,
829 MemorySSAUpdater *MSSAU) {
782830 bool Changed = false;
783831
784832 // If loop header has only one reachable successor we should keep looking for
812860 if (isa(SI->getCondition()))
813861 return Changed;
814862
815 if (!unswitchTrivialSwitch(L, *SI, DT, LI, SE))
863 if (!unswitchTrivialSwitch(L, *SI, DT, LI, SE, MSSAU))
816864 // Couldn't unswitch this one so we're done.
817865 return Changed;
818866
844892
845893 // Found a trivial condition candidate: non-foldable conditional branch. If
846894 // we fail to unswitch this, we can't do anything else that is trivial.
847 if (!unswitchTrivialBranch(L, *BI, DT, LI, SE))
895 if (!unswitchTrivialBranch(L, *BI, DT, LI, SE, MSSAU))
848896 return Changed;
849897
850898 // Mark that we managed to unswitch something.
897945 const SmallDenseMap &DominatingSucc,
898946 ValueToValueMapTy &VMap,
899947 SmallVectorImpl &DTUpdates, AssumptionCache &AC,
900 DominatorTree &DT, LoopInfo &LI) {
948 DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU) {
901949 SmallVector NewBlocks;
902950 NewBlocks.reserve(L.getNumBlocks() + ExitBlocks.size());
903951
942990 // place to merge the CFG, so split the exit first. This is always safe to
943991 // do because there cannot be any non-loop predecessors of a loop exit in
944992 // loop simplified form.
945 auto *MergeBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI);
993 auto *MergeBB = SplitBlock(ExitBB, &ExitBB->front(), &DT, &LI, MSSAU);
946994
947995 // Rearrange the names to make it easier to write test cases by having the
948996 // exit block carry the suffix rather than the merge block carrying the
13731421 static void
13741422 deleteDeadClonedBlocks(Loop &L, ArrayRef ExitBlocks,
13751423 ArrayRef> VMaps,
1376 DominatorTree &DT) {
1424 DominatorTree &DT, MemorySSAUpdater *MSSAU) {
13771425 // Find all the dead clones, and remove them from their successors.
13781426 SmallVector DeadBlocks;
13791427 for (BasicBlock *BB : llvm::concat(L.blocks(), ExitBlocks))
13851433 DeadBlocks.push_back(ClonedBB);
13861434 }
13871435
1436 // Remove all MemorySSA in the dead blocks
1437 if (MSSAU) {
1438 SmallPtrSet DeadBlockSet(DeadBlocks.begin(),
1439 DeadBlocks.end());
1440 MSSAU->removeBlocks(DeadBlockSet);
1441 }
1442
13881443 // Drop any remaining references to break cycles.
13891444 for (BasicBlock *BB : DeadBlocks)
13901445 BB->dropAllReferences();
13931448 BB->eraseFromParent();
13941449 }
13951450
1396 static void
1397 deleteDeadBlocksFromLoop(Loop &L,
1398 SmallVectorImpl &ExitBlocks,
1399 DominatorTree &DT, LoopInfo &LI) {
1451 static void deleteDeadBlocksFromLoop(Loop &L,
1452 SmallVectorImpl &ExitBlocks,
1453 DominatorTree &DT, LoopInfo &LI,
1454 MemorySSAUpdater *MSSAU) {
14001455 // Find all the dead blocks tied to this loop, and remove them from their
14011456 // successors.
14021457 SmallPtrSet DeadBlockSet;
14161471 DeadBlockSet.insert(BB);
14171472 }
14181473 }
1474
1475 // Remove all MemorySSA in the dead blocks
1476 if (MSSAU)
1477 MSSAU->removeBlocks(DeadBlockSet);
14191478
14201479 // Filter out the dead blocks from the exit blocks list so that it can be
14211480 // used in the caller.
18161875 Loop &L, Instruction &TI, ArrayRef Invariants,
18171876 SmallVectorImpl &ExitBlocks, DominatorTree &DT, LoopInfo &LI,
18181877 AssumptionCache &AC, function_ref)> UnswitchCB,
1819 ScalarEvolution *SE) {
1878 ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
18201879 auto *ParentBB = TI.getParent();
18211880 BranchInst *BI = dyn_cast(&TI);
18221881 SwitchInst *SI = BI ? nullptr : cast(&TI);
18321891 else
18331892 assert(isa(BI->getCondition()) &&
18341893 "Partial unswitching requires an instruction as the condition!");
1894
1895 if (MSSAU && VerifyMemorySSA)
1896 MSSAU->getMemorySSA()->verifyMemorySSA();
18351897
18361898 // Constant and BBs tracking the cloned and continuing successor. When we are
18371899 // unswitching the entire condition, this can just be trivially chosen to
18731935
18741936 // Compute the parent loop now before we start hacking on things.
18751937 Loop *ParentL = L.getParentLoop();
1938 // Get blocks in RPO order for MSSA update, before changing the CFG.
1939 LoopBlocksRPO LBRPO(&L);
1940 if (MSSAU)
1941 LBRPO.perform(&LI);
18761942
18771943 // Compute the outer-most loop containing one of our exit blocks. This is the
18781944 // furthest up our loopnest which can be mutated, which we will use below to
19221988 // between the unswitched versions, and we will have a new preheader for the
19231989 // original loop.
19241990 BasicBlock *SplitBB = L.getLoopPreheader();
1925 BasicBlock *LoopPH = SplitEdge(SplitBB, L.getHeader(), &DT, &LI);
1991 BasicBlock *LoopPH = SplitEdge(SplitBB, L.getHeader(), &DT, &LI, MSSAU);
19261992
19271993 // Keep track of the dominator tree updates needed.
19281994 SmallVector DTUpdates;
19352001 VMaps.emplace_back(new ValueToValueMapTy());
19362002 ClonedPHs[SuccBB] = buildClonedLoopBlocks(
19372003 L, LoopPH, SplitBB, ExitBlocks, ParentBB, SuccBB, RetainedSuccBB,
1938 DominatingSucc, *VMaps.back(), DTUpdates, AC, DT, LI);
2004 DominatingSucc, *VMaps.back(), DTUpdates, AC, DT, LI, MSSAU);
19392005 }
19402006
19412007 // The stitching of the branched code back together depends on whether we're
19432009 // nuke the initial terminator placed in the split block.
19442010 SplitBB->getTerminator()->eraseFromParent();
19452011 if (FullUnswitch) {
1946 // First we need to unhook the successor relationship as we'll be replacing
2012 // Splice the terminator from the original loop and rewrite its
2013 // successors.
2014 SplitBB->getInstList().splice(SplitBB->end(), ParentBB->getInstList(), TI);
2015
2016 // Keep a clone of the terminator for MSSA updates.
2017 Instruction *NewTI = TI.clone();
2018 ParentBB->getInstList().push_back(NewTI);
2019
2020 // First wire up the moved terminator to the preheaders.
2021 if (BI) {
2022 BasicBlock *ClonedPH = ClonedPHs.begin()->second;
2023 BI->setSuccessor(ClonedSucc, ClonedPH);
2024 BI->setSuccessor(1 - ClonedSucc, LoopPH);
2025 DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH});
2026 } else {
2027 assert(SI && "Must either be a branch or switch!");
2028
2029 // Walk the cases and directly update their successors.
2030 assert(SI->getDefaultDest() == RetainedSuccBB &&
2031 "Not retaining default successor!");
2032 SI->setDefaultDest(LoopPH);
2033 for (auto &Case : SI->cases())
2034 if (Case.getCaseSuccessor() == RetainedSuccBB)
2035 Case.setSuccessor(LoopPH);
2036 else
2037 Case.setSuccessor(ClonedPHs.find(Case.getCaseSuccessor())->second);
2038
2039 // We need to use the set to populate domtree updates as even when there
2040 // are multiple cases pointing at the same successor we only want to
2041 // remove and insert one edge in the domtree.
2042 for (BasicBlock *SuccBB : UnswitchedSuccBBs)
2043 DTUpdates.push_back(
2044 {DominatorTree::Insert, SplitBB, ClonedPHs.find(SuccBB)->second});
2045 }
2046
2047 if (MSSAU) {
2048 DT.applyUpdates(DTUpdates);
2049 DTUpdates.clear();
2050
2051 // Remove all but one edge to the retained block and all unswitched
2052 // blocks. This is to avoid having duplicate entries in the cloned Phis,
2053 // when we know we only keep a single edge for each case.
2054 MSSAU->removeDuplicatePhiEdgesBetween(ParentBB, RetainedSuccBB);
2055 for (BasicBlock *SuccBB : UnswitchedSuccBBs)
2056 MSSAU->removeDuplicatePhiEdgesBetween(ParentBB, SuccBB);
2057
2058 for (auto &VMap : VMaps)
2059 MSSAU->updateForClonedLoop(LBRPO, ExitBlocks, *VMap,
2060 /*IgnoreIncomingWithNoClones=*/true);
2061 MSSAU->updateExitBlocksForClonedLoop(ExitBlocks, VMaps, DT);
2062
2063 // Remove all edges to unswitched blocks.
2064 for (BasicBlock *SuccBB : UnswitchedSuccBBs)
2065 MSSAU->removeEdge(ParentBB, SuccBB);
2066 }
2067
2068 // Now unhook the successor relationship as we'll be replacing
19472069 // the terminator with a direct branch. This is much simpler for branches
19482070 // than switches so we handle those first.
19492071 if (BI) {
19612083 // is a duplicate edge to the retained successor as the retained successor
19622084 // is always the default successor and as we'll replace this with a direct
19632085 // branch we no longer need the duplicate entries in the PHI nodes.
1964 assert(SI->getDefaultDest() == RetainedSuccBB &&
2086 SwitchInst *NewSI = cast(NewTI);
2087 assert(NewSI->getDefaultDest() == RetainedSuccBB &&
19652088 "Not retaining default successor!");
1966 for (auto &Case : SI->cases())
2089 for (auto &Case : NewSI->cases())
19672090 Case.getCaseSuccessor()->removePredecessor(
19682091 ParentBB,
19692092 /*DontDeleteUselessPHIs*/ true);
19752098 DTUpdates.push_back({DominatorTree::Delete, ParentBB, SuccBB});
19762099 }
19772100
1978 // Now that we've unhooked the successor relationship, splice the terminator
1979 // from the original loop to the split.
1980 SplitBB->getInstList().splice(SplitBB->end(), ParentBB->getInstList(), TI);
1981
1982 // Now wire up the terminator to the preheaders.
1983 if (BI) {
1984 BasicBlock *ClonedPH = ClonedPHs.begin()->second;
1985 BI->setSuccessor(ClonedSucc, ClonedPH);
1986 BI->setSuccessor(1 - ClonedSucc, LoopPH);
1987 DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH});
1988 } else {
1989 assert(SI && "Must either be a branch or switch!");
1990
1991 // Walk the cases and directly update their successors.
1992 SI->setDefaultDest(LoopPH);
1993 for (auto &Case : SI->cases())
1994 if (Case.getCaseSuccessor() == RetainedSuccBB)
1995 Case.setSuccessor(LoopPH);
1996 else
1997 Case.setSuccessor(ClonedPHs.find(Case.getCaseSuccessor())->second);
1998
1999 // We need to use the set to populate domtree updates as even when there
2000 // are multiple cases pointing at the same successor we only want to
2001 // remove and insert one edge in the domtree.
2002 for (BasicBlock *SuccBB : UnswitchedSuccBBs)
2003 DTUpdates.push_back(
2004 {DominatorTree::Insert, SplitBB, ClonedPHs.find(SuccBB)->second});
2005 }
2101 // After MSSAU update, remove the cloned terminator instruction NewTI.
2102 ParentBB->getTerminator()->eraseFromParent();
20062103
20072104 // Create a new unconditional branch to the continuing block (as opposed to
20082105 // the one cloned).
20212118
20222119 // Apply the updates accumulated above to get an up-to-date dominator tree.
20232120 DT.applyUpdates(DTUpdates);
2121 if (!FullUnswitch && MSSAU) {
2122 // Update MSSA for partial unswitch, after DT update.
2123 SmallVector Updates;
2124 Updates.push_back(
2125 {cfg::UpdateKind::Insert, SplitBB, ClonedPHs.begin()->second});
2126 MSSAU->applyInsertUpdates(Updates, DT);
2127 }
20242128
20252129 // Now that we have an accurate dominator tree, first delete the dead cloned
20262130 // blocks so that we can accurately build any cloned loops. It is important to
20272131 // not delete the blocks from the original loop yet because we still want to
20282132 // reference the original loop to understand the cloned loop's structure.
2029 deleteDeadClonedBlocks(L, ExitBlocks, VMaps, DT);
2133 deleteDeadClonedBlocks(L, ExitBlocks, VMaps, DT, MSSAU);
20302134
20312135 // Build the cloned loop structure itself. This may be substantially
20322136 // different from the original structure due to the simplified CFG. This also
20382142 // Now that our cloned loops have been built, we can update the original loop.
20392143 // First we delete the dead blocks from it and then we rebuild the loop
20402144 // structure taking these deletions into account.
2041 deleteDeadBlocksFromLoop(L, ExitBlocks, DT, LI);
2145 deleteDeadBlocksFromLoop(L, ExitBlocks, DT, LI, MSSAU);
2146
2147 if (MSSAU && VerifyMemorySSA)
2148 MSSAU->getMemorySSA()->verifyMemorySSA();
2149
20422150 SmallVector HoistedLoops;
20432151 bool IsStillLoop = rebuildLoopAfterUnswitch(L, ExitBlocks, LI, HoistedLoops);
2152
2153 if (MSSAU && VerifyMemorySSA)
2154 MSSAU->getMemorySSA()->verifyMemorySSA();
20442155
20452156 // This transformation has a high risk of corrupting the dominator tree, and
20462157 // the below steps to rebuild loop structures will result in hard to debug
21662277 SibLoops.push_back(UpdatedL);
21672278 UnswitchCB(IsStillLoop, SibLoops);
21682279
2280 if (MSSAU && VerifyMemorySSA)
2281 MSSAU->getMemorySSA()->verifyMemorySSA();
2282
21692283 ++NumBranches;
21702284 }
21712285
22262340 static BranchInst *
22272341 turnGuardIntoBranch(IntrinsicInst *GI, Loop &L,
22282342 SmallVectorImpl &ExitBlocks,
2229 DominatorTree &DT, LoopInfo &LI) {
2343 DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU) {
22302344 SmallVector DTUpdates;
22312345 LLVM_DEBUG(dbgs() << "Turning " << *GI << " into a branch.\n");
22322346 BasicBlock *CheckBB = GI->getParent();
2347
2348 if (MSSAU && VerifyMemorySSA) {
2349 MSSAU->getMemorySSA()->verifyMemorySSA();
2350 MSSAU->getMemorySSA()->dump();
2351 }
22332352
22342353 // Remove all CheckBB's successors from DomTree. A block can be seen among
22352354 // successors more than once, but for DomTree it should be added only once.
22482367 BasicBlock *GuardedBlock = CheckBI->getSuccessor(0);
22492368 GuardedBlock->setName("guarded");
22502369 CheckBI->getSuccessor(1)->setName("deopt");
2370 BasicBlock *DeoptBlock = CheckBI->getSuccessor(1);
22512371
22522372 // We now have a new exit block.
22532373 ExitBlocks.push_back(CheckBI->getSuccessor(1));
2374
2375 if (MSSAU)
2376 MSSAU->moveAllAfterSpliceBlocks(CheckBB, GuardedBlock, GI);
22542377
22552378 GI->moveBefore(DeoptBlockTerm);
22562379 GI->setArgOperand(0, ConstantInt::getFalse(GI->getContext()));
22682391 DT.applyUpdates(DTUpdates);
22692392 // Inform LI of a new loop block.
22702393 L.addBasicBlockToLoop(GuardedBlock, LI);
2394
2395 if (MSSAU) {
2396 MemoryDef *MD = cast(MSSAU->getMemorySSA()->getMemoryAccess(GI));
2397 MSSAU->moveToPlace(MD, DeoptBlock, MemorySSA::End);
2398 if (VerifyMemorySSA)
2399 MSSAU->getMemorySSA()->verifyMemorySSA();
2400 }
22712401
22722402 ++NumGuards;
22732403 return CheckBI;
23622492 unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
23632493 AssumptionCache &AC, TargetTransformInfo &TTI,
23642494 function_ref)> UnswitchCB,
2365 ScalarEvolution *SE) {
2495 ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
23662496 // Collect all invariant conditions within this loop (as opposed to an inner
23672497 // loop which would be handled when visiting that inner loop).
23682498 SmallVector>, 4>
26042734 // If the best candidate is a guard, turn it into a branch.
26052735 if (isGuard(BestUnswitchTI))
26062736 BestUnswitchTI = turnGuardIntoBranch(cast(BestUnswitchTI), L,
2607 ExitBlocks, DT, LI);
2737 ExitBlocks, DT, LI, MSSAU);
26082738
26092739 LLVM_DEBUG(dbgs() << " Unswitching non-trivial (cost = "
26102740 << BestUnswitchCost << ") terminator: " << *BestUnswitchTI
26112741 << "\n");
26122742 unswitchNontrivialInvariants(L, *BestUnswitchTI, BestUnswitchInvariants,
2613 ExitBlocks, DT, LI, AC, UnswitchCB, SE);
2743 ExitBlocks, DT, LI, AC, UnswitchCB, SE, MSSAU);
26142744 return true;
26152745 }
26162746
26232753 ///
26242754 /// The `DT`, `LI`, `AC`, `TTI` parameters are required analyses that are also
26252755 /// updated based on the unswitch.
2756 /// The `MSSA` analysis is also updated if valid (i.e. its use is enabled).
26262757 ///
26272758 /// If either `NonTrivial` is true or the flag `EnableNonTrivialUnswitch` is
26282759 /// true, we will attempt to do non-trivial unswitching as well as trivial
26382769 AssumptionCache &AC, TargetTransformInfo &TTI,
26392770 bool NonTrivial,
26402771 function_ref)> UnswitchCB,
2641 ScalarEvolution *SE) {
2772 ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
26422773 assert(L.isRecursivelyLCSSAForm(DT, LI) &&
26432774 "Loops must be in LCSSA form before unswitching.");
26442775 bool Changed = false;
26482779 return false;
26492780
26502781 // Try trivial unswitch first before loop over other basic blocks in the loop.
2651 if (unswitchAllTrivialConditions(L, DT, LI, SE)) {
2782 if (unswitchAllTrivialConditions(L, DT, LI, SE, MSSAU)) {
26522783 // If we unswitched successfully we will want to clean up the loop before
26532784 // processing it further so just mark it as unswitched and return.
26542785 UnswitchCB(/*CurrentLoopValid*/ true, {});
26692800
26702801 // Try to unswitch the best invariant condition. We prefer this full unswitch to
26712802 // a partial unswitch when possible below the threshold.
2672 if (unswitchBestCondition(L, DT, LI, AC, TTI, UnswitchCB, SE))
2803 if (unswitchBestCondition(L, DT, LI, AC, TTI, UnswitchCB, SE, MSSAU))
26732804 return true;
26742805
26752806 // No other opportunities to unswitch.
27032834 U.markLoopAsDeleted(L, LoopName);
27042835 };
27052836
2837 Optional MSSAU;
2838 if (AR.MSSA) {
2839 MSSAU = MemorySSAUpdater(AR.MSSA);
2840 if (VerifyMemorySSA)
2841 AR.MSSA->verifyMemorySSA();
2842 }
27062843 if (!unswitchLoop(L, AR.DT, AR.LI, AR.AC, AR.TTI, NonTrivial, UnswitchCB,
2707 &AR.SE))
2844 &AR.SE, MSSAU.hasValue() ? MSSAU.getPointer() : nullptr))
27082845 return PreservedAnalyses::all();
2846
2847 if (AR.MSSA && VerifyMemorySSA)
2848 AR.MSSA->verifyMemorySSA();
27092849
27102850 // Historically this pass has had issues with the dominator tree so verify it
27112851 // in asserts builds.
27322872 void getAnalysisUsage(AnalysisUsage &AU) const override {
27332873 AU.addRequired();
27342874 AU.addRequired();
2875 if (EnableMSSALoopDependency) {
2876 AU.addRequired();
2877 AU.addPreserved();
2878 }
27352879 getLoopAnalysisUsage(AU);
27362880 }
27372881 };
27512895 auto &LI = getAnalysis().getLoopInfo();
27522896 auto &AC = getAnalysis().getAssumptionCache(F);
27532897 auto &TTI = getAnalysis().getTTI(F);
2898 MemorySSA *MSSA = nullptr;
2899 Optional MSSAU;
2900 if (EnableMSSALoopDependency) {
2901 MSSA = &getAnalysis().getMSSA();
2902 MSSAU = MemorySSAUpdater(MSSA);
2903 }
27542904
27552905 auto *SEWP = getAnalysisIfAvailable();
27562906 auto *SE = SEWP ? &SEWP->getSE() : nullptr;
27702920 LPM.markLoopAsDeleted(*L);
27712921 };
27722922
2773 bool Changed = unswitchLoop(*L, DT, LI, AC, TTI, NonTrivial, UnswitchCB, SE);
2923 if (MSSA && VerifyMemorySSA)
2924 MSSA->verifyMemorySSA();
2925
2926 bool Changed = unswitchLoop(*L, DT, LI, AC, TTI, NonTrivial, UnswitchCB, SE,
2927 MSSAU.hasValue() ? MSSAU.getPointer() : nullptr);
2928
2929 if (MSSA && VerifyMemorySSA)
2930 MSSA->verifyMemorySSA();
27742931
27752932 // If anything was unswitched, also clear any cached information about this
27762933 // loop.
27902947 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
27912948 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
27922949 INITIALIZE_PASS_DEPENDENCY(LoopPass)
2950 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
27932951 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
27942952 INITIALIZE_PASS_END(SimpleLoopUnswitchLegacyPass, "simple-loop-unswitch",
27952953 "Simple unswitch loops", false, false)
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12
23 %struct.BLEND_MAP = type { i16, i16, i16, i32, %struct.BLEND_MAP_ENTRY* }
34 %struct.BLEND_MAP_ENTRY = type { float, i8, { [5 x float], [4 x i8] } }
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12
23 define void @init_caller_save() {
34 entry:
0 ; PR1333
11 ; RUN: opt < %s -simple-loop-unswitch -disable-output
2 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
23
34 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
45 target triple = "i686-pc-linux-gnu"
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12 ; PR1333
23
34 define void @pp_cxx_expression() {
0 ; RUN: opt < %s -simple-loop-unswitch -instcombine -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
12
23 @str3 = external constant [3 x i8] ; <[3 x i8]*> [#uses=1]
34
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12
23 define i32 @main(i32 %argc, i8** %argv) {
34 entry:
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12 ; PR1559
23
34 target triple = "i686-pc-linux-gnu"
0 ; RUN: opt < %s -simple-loop-unswitch -instcombine -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
12 %struct.ClassDef = type { %struct.QByteArray, %struct.QByteArray, %"struct.QList", %"struct.QList", i8, i8, %"struct.QList", %"struct.QList", %"struct.QList", %"struct.QList", %"struct.QList", %"struct.QList", %"struct.QMap", %"struct.QList", %"struct.QMap", i32, i32 }
23 %struct.FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
34 %struct.Generator = type { %struct.FILE*, %struct.ClassDef*, %"struct.QList", %struct.QByteArray, %"struct.QList" }
0 ; RUN: opt < %s -simple-loop-unswitch -instcombine -gvn -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -gvn -disable-output
12 ; PR2372
23 target triple = "i386-pc-linux-gnu"
34
0 ; RUN: opt < %s -simple-loop-unswitch
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa
12 ; PR8622
23 @g_38 = external global i32, align 4
34
0 ; RUN: opt -simple-loop-unswitch -disable-output < %s
1 ; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output < %s
12 ; PR10031
23
34 define i32 @test(i32 %command) {
0 ; RUN: opt < %s -sroa -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -sroa -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12 ; PR11016
23 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
34 target triple = "x86_64-apple-macosx10.7.2"
0 ; RUN: opt < %s -S -simple-loop-unswitch -verify-loop-info -verify-dom-info | FileCheck %s
1 ; RUN: opt < %s -S -simple-loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
12 ; PR12343: -simple-loop-unswitch crash on indirect branch
23
34 ; CHECK: %0 = icmp eq i64 undef, 0
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12 ; PR12887
23 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
34 target triple = "x86_64-unknown-linux-gnu"
0 ; RUN: opt < %s -simple-loop-unswitch -S | FileCheck %s
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
12
23 ; In cases where two address spaces do not have the same size pointer, the
34 ; input for the addrspacecast should not be used as a substitute for itself
0 ; RUN: opt < %s -simple-loop-unswitch -S 2>&1 | FileCheck %s
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S 2>&1 | FileCheck %s
12
23 ; This is to test trivial loop unswitch only happens when trivial condition
34 ; itself is an LIV loop condition (not partial LIV which could occur in and/or).
0 ; RUN: opt -passes='loop(unswitch),verify' -S < %s | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify' -S < %s | FileCheck %s
12
23 define i32 @test(i32* %A, i1 %C) {
34 entry:
0 ; RUN: opt -S -simple-loop-unswitch < %s | FileCheck %s
1 ; RUN: opt -S -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
12 target triple = "x86_64-pc-win32"
23
34 define void @f(i32 %doit, i1 %x, i1 %y) personality i32 (...)* @__CxxFrameHandler3 {
0 ; RUN: opt < %s -simple-loop-unswitch -S | FileCheck %s
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
12
23 ; This test checks if unswitched condition preserve make.implicit metadata.
34 define i32 @test(i1 %cond) {
0 ; RUN: opt < %s -simple-loop-unswitch -disable-output
1 ; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
12
23 define void @test1(i32* %S2) {
34 entry:
0 ; RUN: opt -simple-loop-unswitch -S < %s | FileCheck %s
1 ; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
12
23 define void @f(i32 %n, i32* %ptr) {
34 ; CHECK-LABEL: @f(
0 ; RUN: opt -passes='loop(unswitch),verify' -enable-nontrivial-unswitch -simple-loop-unswitch-guards -S < %s | FileCheck %s
11 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -simple-loop-unswitch-guards -S < %s | FileCheck %s
2 ; RUN: opt -passes='loop(unswitch),verify' -enable-nontrivial-unswitch -simple-loop-unswitch-guards -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
23
34 declare void @llvm.experimental.guard(i1, ...)
45
0 ; REQUIRES: asserts
11 ; RUN: opt -simple-loop-unswitch -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
22 ; RUN: opt -simple-loop-unswitch -S < %s | FileCheck %s
3 ; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
34 ; PR5373
45
56 ; Loop unswitching shouldn't trivially unswitch the true case of condition %a
0 ; RUN: opt -passes='loop(unswitch),verify' -S < %s | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify' -S < %s | FileCheck %s
12
23 declare void @unknown()
34 declare void @unknown2()
11 ;
22 ; RUN: opt -passes='loop(unswitch),verify' -enable-nontrivial-unswitch -unswitch-threshold=5 -S < %s | FileCheck %s
33 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -S < %s | FileCheck %s
4 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
45
56 declare void @a()
67 declare void @b()
0 ; RUN: opt -passes='loop(unswitch),verify' -enable-nontrivial-unswitch -S < %s | FileCheck %s
11 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -S < %s | FileCheck %s
2 ; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
23
34 declare i32 @a()
45 declare i32 @b()
0 ; RUN: opt -simple-loop-unswitch -loop-deletion -S < %s | FileCheck %s
1 ; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -loop-deletion -S < %s | FileCheck %s
12 ;
23 ; Check that when we do unswitching where we re-enqueue the loop to be processed
34 ; again, but manage to delete the loop before ever getting to iterate on it, it
0 ; RUN: opt -simple-loop-unswitch -verify-loop-info -verify-dom-info -disable-output < %s
1 ; RUN: opt -simple-loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output < %s
12
23 ; Loop unswitch should be able to unswitch these loops and
34 ; preserve LCSSA and LoopSimplify forms.
0 ; RUN: opt -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify' -S < %s | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify' -S < %s | FileCheck %s
12
23 declare void @some_func() noreturn
34
0 ; RUN: opt -passes='loop(unswitch),verify' -S < %s | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify' -S < %s | FileCheck %s
12
23 declare void @some_func() noreturn
34 declare void @sink(i32)
0 ; RUN: opt -passes='print,loop(unswitch,loop-instsimplify),print' -enable-nontrivial-unswitch -S < %s 2>%t.scev | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='print,loop(unswitch,loop-instsimplify),print' -enable-nontrivial-unswitch -S < %s 2>%t.scev | FileCheck %s
12 ; RUN: FileCheck %s --check-prefix=SCEV < %t.scev
23
34 target triple = "x86_64-unknown-linux-gnu"