llvm.org GIT mirror llvm / 33ad346
Use MemorySSA in LICM to do sinking and hoisting. Summary: Step 2 in using MemorySSA in LICM: Use MemorySSA in LICM to do sinking and hoisting, all under "EnableMSSALoopDependency" flag. Promotion is disabled. Enable flag in LICM sink/hoist tests to test correctness of this change. Moved one test which relied on promotion, in order to test all sinking tests. Reviewers: sanjoy, davide, gberry, george.burgess.iv Subscribers: llvm-commits, Prazek Differential Revision: https://reviews.llvm.org/D40375 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@350879 91177308-0d34-0410-b5e6-96231b3b80d8 Alina Sbirlea 8 months ago
18 changed file(s) with 359 addition(s) and 161 deletion(s). Raw diff Collapse all Expand all
3434 #include "llvm/ADT/SmallPtrSet.h"
3535 #include "llvm/ADT/SmallSet.h"
3636 #include "llvm/ADT/SmallVector.h"
37 #include "llvm/Analysis/LoopInfo.h"
3738 #include "llvm/Analysis/LoopIterator.h"
3839 #include "llvm/Analysis/MemorySSA.h"
3940 #include "llvm/IR/BasicBlock.h"
4040 class DataLayout;
4141 class Loop;
4242 class LoopInfo;
43 class MemorySSAUpdater;
4344 class OptimizationRemarkEmitter;
4445 class PredicatedScalarEvolution;
4546 class PredIteratorCache;
108109 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
109110 bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
110111 TargetLibraryInfo *, TargetTransformInfo *, Loop *,
111 AliasSetTracker *, ICFLoopSafetyInfo *,
112 AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
112113 OptimizationRemarkEmitter *ORE);
113114
114115 /// Walk the specified region of the CFG (defined by all blocks
121122 /// ORE. It returns changed status.
122123 bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
123124 TargetLibraryInfo *, Loop *, AliasSetTracker *,
124 ICFLoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
125 MemorySSAUpdater *, ICFLoopSafetyInfo *,
126 OptimizationRemarkEmitter *ORE);
125127
126128 /// This function deletes dead loops. The caller of this function needs to
127129 /// guarantee that the loop is infact dead.
273275 /// If \p ORE is set use it to emit optimization remarks.
274276 bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
275277 Loop *CurLoop, AliasSetTracker *CurAST,
276 bool TargetExecutesOncePerLoop,
278 MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop,
277279 OptimizationRemarkEmitter *ORE = nullptr);
278280
279281 /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
4545 #include "llvm/Analysis/LoopPass.h"
4646 #include "llvm/Analysis/MemoryBuiltins.h"
4747 #include "llvm/Analysis/MemorySSA.h"
48 #include "llvm/Analysis/MemorySSAUpdater.h"
4849 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
4950 #include "llvm/Analysis/ScalarEvolution.h"
5051 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
5152 #include "llvm/Analysis/TargetLibraryInfo.h"
52 #include "llvm/Transforms/Utils/Local.h"
5353 #include "llvm/Analysis/ValueTracking.h"
5454 #include "llvm/IR/CFG.h"
5555 #include "llvm/IR/Constants.h"
6868 #include "llvm/Transforms/Scalar.h"
6969 #include "llvm/Transforms/Scalar/LoopPassManager.h"
7070 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
71 #include "llvm/Transforms/Utils/Local.h"
7172 #include "llvm/Transforms/Utils/LoopUtils.h"
7273 #include "llvm/Transforms/Utils/SSAUpdater.h"
7374 #include
105106 LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
106107 cl::desc("How many instruction to cross product using AA"));
107108
109 // Experimental option to allow imprecision in LICM (use MemorySSA cap) in
110 // pathological cases, in exchange for faster compile. This is to be removed
111 // if MemorySSA starts to address the same issue. This flag applies only when
112 // LICM uses MemorySSA instead on AliasSetTracker. When the flag is disabled
113 // (default), LICM calls MemorySSAWalker's getClobberingMemoryAccess, which
114 // gets perfect accuracy. When flag is enabled, LICM will call into MemorySSA's
115 // getDefiningAccess, which may not be precise, since optimizeUses is capped.
116 static cl::opt EnableLicmCap(
117 "enable-licm-cap", cl::init(false), cl::Hidden,
118 cl::desc("Enable imprecision in LICM (uses MemorySSA cap) in "
119 "pathological cases, in exchange for faster compile"));
120
108121 static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
109122 static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
110123 const LoopSafetyInfo *SafetyInfo,
111124 TargetTransformInfo *TTI, bool &FreeInLoop);
112125 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
113126 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
114 OptimizationRemarkEmitter *ORE);
127 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE);
115128 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
116129 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
117 OptimizationRemarkEmitter *ORE, bool FreeInLoop);
130 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE,
131 bool FreeInLoop);
118132 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
119133 const DominatorTree *DT,
120134 const Loop *CurLoop,
124138 static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
125139 AliasSetTracker *CurAST, Loop *CurLoop,
126140 AliasAnalysis *AA);
127
128 static Instruction *
129 CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
130 const LoopInfo *LI,
131 const LoopSafetyInfo *SafetyInfo);
141 static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
142 Loop *CurLoop);
143 static Instruction *CloneInstructionInExitBlock(
144 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
145 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
132146
133147 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
134 AliasSetTracker *AST);
148 AliasSetTracker *AST, MemorySSAUpdater *MSSAU);
135149
136150 static void moveInstructionBefore(Instruction &I, Instruction &Dest,
137151 ICFLoopSafetyInfo &SafetyInfo);
193207 AU.addPreserved();
194208 AU.addPreserved();
195209 AU.addRequired();
196 if (EnableMSSALoopDependency)
210 if (EnableMSSALoopDependency) {
197211 AU.addRequired();
212 AU.addPreserved();
213 }
198214 AU.addRequired();
199215 getLoopAnalysisUsage(AU);
200216 }
274290
275291 assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.");
276292
277 std::unique_ptr CurAST = collectAliasInfoForLoop(L, LI, AA);
293 std::unique_ptr CurAST;
294 std::unique_ptr MSSAU;
295 if (!MSSA) {
296 LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n");
297 CurAST = collectAliasInfoForLoop(L, LI, AA);
298 } else {
299 LLVM_DEBUG(dbgs() << "LICM: Using MemorySSA. Promotion disabled.\n");
300 MSSAU = make_unique(MSSA);
301 }
278302
279303 // Get the preheader block to move instructions into...
280304 BasicBlock *Preheader = L->getLoopPreheader();
295319 //
296320 if (L->hasDedicatedExits())
297321 Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, TTI, L,
298 CurAST.get(), &SafetyInfo, ORE);
322 CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
299323 if (Preheader)
300324 Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L,
301 CurAST.get(), &SafetyInfo, ORE);
325 CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
302326
303327 // Now that all loop invariants have been removed from the loop, promote any
304328 // memory references to scalars that we can.
327351
328352 bool Promoted = false;
329353
330 // Loop over all of the alias sets in the tracker object.
331 for (AliasSet &AS : *CurAST) {
332 // We can promote this alias set if it has a store, if it is a "Must"
333 // alias set, if the pointer is loop invariant, and if we are not
334 // eliminating any volatile loads or stores.
335 if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
336 !L->isLoopInvariant(AS.begin()->getValue()))
337 continue;
338
339 assert(
340 !AS.empty() &&
341 "Must alias set should have at least one pointer element in it!");
342
343 SmallSetVector PointerMustAliases;
344 for (const auto &ASI : AS)
345 PointerMustAliases.insert(ASI.getValue());
346
347 Promoted |= promoteLoopAccessesToScalars(
348 PointerMustAliases, ExitBlocks, InsertPts, PIC, LI, DT, TLI, L,
349 CurAST.get(), &SafetyInfo, ORE);
354 if (CurAST.get()) {
355 // Loop over all of the alias sets in the tracker object.
356 for (AliasSet &AS : *CurAST) {
357 // We can promote this alias set if it has a store, if it is a "Must"
358 // alias set, if the pointer is loop invariant, and if we are not
359 // eliminating any volatile loads or stores.
360 if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
361 !L->isLoopInvariant(AS.begin()->getValue()))
362 continue;
363
364 assert(
365 !AS.empty() &&
366 "Must alias set should have at least one pointer element in it!");
367
368 SmallSetVector PointerMustAliases;
369 for (const auto &ASI : AS)
370 PointerMustAliases.insert(ASI.getValue());
371
372 Promoted |= promoteLoopAccessesToScalars(
373 PointerMustAliases, ExitBlocks, InsertPts, PIC, LI, DT, TLI, L,
374 CurAST.get(), &SafetyInfo, ORE);
375 }
350376 }
377 // FIXME: Promotion initially disabled when using MemorySSA.
351378
352379 // Once we have promoted values across the loop body we have to
353380 // recursively reform LCSSA as any nested loop may now have values defined
371398
372399 // If this loop is nested inside of another one, save the alias information
373400 // for when we process the outer loop.
374 if (L->getParentLoop() && !DeleteAST)
401 if (CurAST.get() && L->getParentLoop() && !DeleteAST)
375402 LoopToAliasSetMap[L] = std::move(CurAST);
403
404 if (MSSAU.get() && VerifyMemorySSA)
405 MSSAU->getMemorySSA()->verifyMemorySSA();
376406
377407 if (Changed && SE)
378408 SE->forgetLoopDispositions(L);
387417 bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
388418 DominatorTree *DT, TargetLibraryInfo *TLI,
389419 TargetTransformInfo *TTI, Loop *CurLoop,
390 AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
420 AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
421 ICFLoopSafetyInfo *SafetyInfo,
391422 OptimizationRemarkEmitter *ORE) {
392423
393424 // Verify inputs.
394425 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
395 CurLoop != nullptr && CurAST && SafetyInfo != nullptr &&
396 "Unexpected input to sinkRegion");
426 CurLoop != nullptr && SafetyInfo != nullptr &&
427 "Unexpected input to sinkRegion.");
428 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&
429 "Either AliasSetTracker or MemorySSA should be initialized.");
397430
398431 // We want to visit children before parents. We will enque all the parents
399432 // before their children in the worklist and process the worklist in reverse
417450 LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n');
418451 salvageDebugInfo(I);
419452 ++II;
420 eraseInstruction(I, *SafetyInfo, CurAST);
453 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
421454 Changed = true;
422455 continue;
423456 }
429462 //
430463 bool FreeInLoop = false;
431464 if (isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
432 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
465 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, ORE) &&
433466 !I.mayHaveSideEffects()) {
434 if (sink(I, LI, DT, CurLoop, SafetyInfo, ORE, FreeInLoop)) {
467 if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE, FreeInLoop)) {
435468 if (!FreeInLoop) {
436469 ++II;
437 eraseInstruction(I, *SafetyInfo, CurAST);
470 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
438471 }
439472 Changed = true;
440473 }
441474 }
442475 }
443476 }
477 if (MSSAU && VerifyMemorySSA)
478 MSSAU->getMemorySSA()->verifyMemorySSA();
444479 return Changed;
445480 }
446481
457492 LoopInfo *LI;
458493 DominatorTree *DT;
459494 Loop *CurLoop;
495 MemorySSAUpdater *MSSAU;
460496
461497 // A map of blocks in the loop to the block their instructions will be hoisted
462498 // to.
467503 DenseMap HoistableBranches;
468504
469505 public:
470 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop)
471 : LI(LI), DT(DT), CurLoop(CurLoop) {}
506 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
507 MemorySSAUpdater *MSSAU)
508 : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}
472509
473510 void registerPossiblyHoistableBranch(BranchInst *BI) {
474511 // We can only hoist conditional branches with loop invariant operands.
643680 if (HoistTarget == InitialPreheader) {
644681 // Phis in the loop header now need to use the new preheader.
645682 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
683 if (MSSAU)
684 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
685 HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
646686 // The new preheader dominates the loop header.
647687 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
648688 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
673713 ///
674714 bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
675715 DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
676 AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
716 AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
717 ICFLoopSafetyInfo *SafetyInfo,
677718 OptimizationRemarkEmitter *ORE) {
678719 // Verify inputs.
679720 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
680 CurLoop != nullptr && CurAST != nullptr && SafetyInfo != nullptr &&
681 "Unexpected input to hoistRegion");
682
683 ControlFlowHoister CFH(LI, DT, CurLoop);
721 CurLoop != nullptr && SafetyInfo != nullptr &&
722 "Unexpected input to hoistRegion.");
723 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&
724 "Either AliasSetTracker or MemorySSA should be initialized.");
725
726 ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);
684727
685728 // Keep track of instructions that have been hoisted, as they may need to be
686729 // re-hoisted if they end up not dominating all of their uses.
707750 &I, I.getModule()->getDataLayout(), TLI)) {
708751 LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << " --> " << *C
709752 << '\n');
710 CurAST->copyValue(&I, C);
753 if (CurAST)
754 CurAST->copyValue(&I, C);
755 // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
711756 I.replaceAllUsesWith(C);
712757 if (isInstructionTriviallyDead(&I, TLI))
713 eraseInstruction(I, *SafetyInfo, CurAST);
758 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
714759 Changed = true;
715760 continue;
716761 }
722767 // and we have accurately duplicated the control flow from the loop header
723768 // to that block.
724769 if (CurLoop->hasLoopInvariantOperands(&I) &&
725 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
770 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, ORE) &&
726771 isSafeToExecuteUnconditionally(
727772 I, DT, CurLoop, SafetyInfo, ORE,
728773 CurLoop->getLoopPreheader()->getTerminator())) {
729 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
774 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
775 MSSAU, ORE);
730776 HoistedInstructions.push_back(&I);
731777 Changed = true;
732778 continue;
750796 SafetyInfo->insertInstructionTo(Product, I.getParent());
751797 Product->insertAfter(&I);
752798 I.replaceAllUsesWith(Product);
753 eraseInstruction(I, *SafetyInfo, CurAST);
799 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
754800
755801 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
756 SafetyInfo, ORE);
802 SafetyInfo, MSSAU, ORE);
757803 HoistedInstructions.push_back(ReciprocalDivisor);
758804 Changed = true;
759805 continue;
766812 CurLoop->hasLoopInvariantOperands(&I) &&
767813 SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
768814 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) {
769 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
815 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
816 MSSAU, ORE);
770817 HoistedInstructions.push_back(&I);
771818 Changed = true;
772819 continue;
780827 PN->setIncomingBlock(
781828 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
782829 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
783 ORE);
830 MSSAU, ORE);
784831 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected");
785832 Changed = true;
786833 continue;
823870 }
824871 }
825872 }
826
827 // Now that we've finished hoisting make sure that LI and DT are still valid.
873 if (MSSAU && VerifyMemorySSA)
874 MSSAU->getMemorySSA()->verifyMemorySSA();
875
876 // Now that we've finished hoisting make sure that LI and DT are still
877 // valid.
828878 #ifndef NDEBUG
829879 if (Changed) {
830880 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
904954 isa(I) || isa(I));
905955 }
906956 /// Return true if all of the alias sets within this AST are known not to
907 /// contain a Mod.
908 bool isReadOnly(AliasSetTracker *CurAST) {
909 for (AliasSet &AS : *CurAST) {
910 if (!AS.isForwardingAliasSet() && AS.isMod()) {
911 return false;
912 }
913 }
957 /// contain a Mod, or if MSSA knows thare are no MemoryDefs in the loop.
958 bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
959 const Loop *L) {
960 if (CurAST) {
961 for (AliasSet &AS : *CurAST) {
962 if (!AS.isForwardingAliasSet() && AS.isMod()) {
963 return false;
964 }
965 }
966 return true;
967 } else { /*MSSAU*/
968 for (auto *BB : L->getBlocks())
969 if (MSSAU->getMemorySSA()->getBlockDefs(BB))
970 return false;
971 return true;
972 }
973 }
974
975 /// Return true if I is the only Instruction with a MemoryAccess in L.
976 bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
977 const MemorySSAUpdater *MSSAU) {
978 for (auto *BB : L->getBlocks())
979 if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
980 int NotAPhi = 0;
981 for (const auto &Acc : *Accs) {
982 if (isa(&Acc))
983 continue;
984 const auto *MUD = cast(&Acc);
985 if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
986 return false;
987 }
988 }
914989 return true;
915990 }
916991 }
917992
918993 bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
919994 Loop *CurLoop, AliasSetTracker *CurAST,
995 MemorySSAUpdater *MSSAU,
920996 bool TargetExecutesOncePerLoop,
921997 OptimizationRemarkEmitter *ORE) {
922998 // If we don't understand the instruction, bail early.
923999 if (!isHoistableAndSinkableInst(I))
9241000 return false;
925
1001
1002 MemorySSA *MSSA = MSSAU ? MSSAU->getMemorySSA() : nullptr;
1003
9261004 // Loads have extra constraints we have to verify before we can hoist them.
9271005 if (LoadInst *LI = dyn_cast(&I)) {
9281006 if (!LI->isUnordered())
9421020 if (isLoadInvariantInLoop(LI, DT, CurLoop))
9431021 return true;
9441022
945 bool Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI),
946 CurAST, CurLoop, AA);
1023 bool Invalidated;
1024 if (CurAST)
1025 Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
1026 CurLoop, AA);
1027 else
1028 Invalidated = pointerInvalidatedByLoopWithMSSA(
1029 MSSA, cast(MSSA->getMemoryAccess(LI)), CurLoop);
9471030 // Check loop-invariant address because this may also be a sinkable load
9481031 // whose address is not necessarily loop-invariant.
9491032 if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
9681051 if (match(CI, m_Intrinsic()))
9691052 // Assumes don't actually alias anything or throw
9701053 return true;
971
1054
9721055 // Handle simple cases by querying alias analysis.
9731056 FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
9741057 if (Behavior == FMRB_DoesNotAccessMemory)
9801063 if (AliasAnalysis::onlyAccessesArgPointees(Behavior)) {
9811064 // TODO: expand to writeable arguments
9821065 for (Value *Op : CI->arg_operands())
983 if (Op->getType()->isPointerTy() &&
984 pointerInvalidatedByLoop(
1066 if (Op->getType()->isPointerTy()) {
1067 bool Invalidated;
1068 if (CurAST)
1069 Invalidated = pointerInvalidatedByLoop(
9851070 MemoryLocation(Op, LocationSize::unknown(), AAMDNodes()),
986 CurAST, CurLoop, AA))
987 return false;
1071 CurAST, CurLoop, AA);
1072 else
1073 Invalidated = pointerInvalidatedByLoopWithMSSA(
1074 MSSA, cast(MSSA->getMemoryAccess(CI)), CurLoop);
1075 if (Invalidated)
1076 return false;
1077 }
9881078 return true;
9891079 }
9901080
9911081 // If this call only reads from memory and there are no writes to memory
9921082 // in the loop, we can hoist or sink the call as appropriate.
993 if (isReadOnly(CurAST))
1083 if (isReadOnly(CurAST, MSSAU, CurLoop))
9941084 return true;
9951085 }
9961086
10011091 } else if (auto *FI = dyn_cast(&I)) {
10021092 // Fences alias (most) everything to provide ordering. For the moment,
10031093 // just give up if there are any other memory operations in the loop.
1004 auto Begin = CurAST->begin();
1005 assert(Begin != CurAST->end() && "must contain FI");
1006 if (std::next(Begin) != CurAST->end())
1007 // constant memory for instance, TODO: handle better
1008 return false;
1009 auto *UniqueI = Begin->getUniqueInstruction();
1010 if (!UniqueI)
1011 // other memory op, give up
1012 return false;
1013 (void)FI; //suppress unused variable warning
1014 assert(UniqueI == FI && "AS must contain FI");
1015 return true;
1094 if (CurAST) {
1095 auto Begin = CurAST->begin();
1096 assert(Begin != CurAST->end() && "must contain FI");
1097 if (std::next(Begin) != CurAST->end())
1098 // constant memory for instance, TODO: handle better
1099 return false;
1100 auto *UniqueI = Begin->getUniqueInstruction();
1101 if (!UniqueI)
1102 // other memory op, give up
1103 return false;
1104 (void)FI; // suppress unused variable warning
1105 assert(UniqueI == FI && "AS must contain FI");
1106 return true;
1107 } else // MSSAU
1108 return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
10161109 } else if (auto *SI = dyn_cast(&I)) {
10171110 if (!SI->isUnordered())
10181111 return false; // Don't sink/hoist volatile or ordered atomic store!
10221115 // load store promotion instead. TODO: We can extend this to cases where
10231116 // there is exactly one write to the location and that write dominates an
10241117 // arbitrary number of reads in the loop.
1025 auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
1026
1027 if (AS.isRef() || !AS.isMustAlias())
1028 // Quick exit test, handled by the full path below as well.
1118 if (CurAST) {
1119 auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
1120
1121 if (AS.isRef() || !AS.isMustAlias())
1122 // Quick exit test, handled by the full path below as well.
1123 return false;
1124 auto *UniqueI = AS.getUniqueInstruction();
1125 if (!UniqueI)
1126 // other memory op, give up
1127 return false;
1128 assert(UniqueI == SI && "AS must contain SI");
1129 return true;
1130 } else { // MSSAU
1131 if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
1132 return true;
1133 if (!EnableLicmCap) {
1134 auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
1135 if (MSSA->isLiveOnEntryDef(Source) ||
1136 !CurLoop->contains(Source->getBlock()))
1137 return true;
1138 }
10291139 return false;
1030 auto *UniqueI = AS.getUniqueInstruction();
1031 if (!UniqueI)
1032 // other memory op, give up
1033 return false;
1034 assert(UniqueI == SI && "AS must contain SI");
1035 return true;
1140 }
10361141 }
10371142
10381143 assert(!I.mayReadOrWriteMemory() && "unhandled aliasing");
11161221 return true;
11171222 }
11181223
1119 static Instruction *
1120 CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
1121 const LoopInfo *LI,
1122 const LoopSafetyInfo *SafetyInfo) {
1224 static Instruction *CloneInstructionInExitBlock(
1225 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
1226 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
11231227 Instruction *New;
11241228 if (auto *CI = dyn_cast(&I)) {
11251229 const auto &BlockColors = SafetyInfo->getBlockColors();
11541258 ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
11551259 if (!I.getName().empty())
11561260 New->setName(I.getName() + ".le");
1261
1262 MemoryAccess *OldMemAcc;
1263 if (MSSAU && (OldMemAcc = MSSAU->getMemorySSA()->getMemoryAccess(&I))) {
1264 // Create a new MemoryAccess and let MemorySSA set its defining access.
1265 MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
1266 New, nullptr, New->getParent(), MemorySSA::Beginning);
1267 if (NewMemAcc) {
1268 if (auto *MemDef = dyn_cast(NewMemAcc))
1269 MSSAU->insertDef(MemDef, /*RenameUses=*/true);
1270 else {
1271 auto *MemUse = cast(NewMemAcc);
1272 MSSAU->insertUse(MemUse);
1273 }
1274 }
1275 }
11571276
11581277 // Build LCSSA PHI nodes for any in-loop operands. Note that this is
11591278 // particularly cheap because we can rip off the PHI node that we're
11781297 }
11791298
11801299 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
1181 AliasSetTracker *AST) {
1300 AliasSetTracker *AST, MemorySSAUpdater *MSSAU) {
11821301 if (AST)
11831302 AST->deleteValue(&I);
1303 if (MSSAU)
1304 MSSAU->removeMemoryAccess(&I);
11841305 SafetyInfo.removeInstruction(&I);
11851306 I.eraseFromParent();
11861307 }
11951316 static Instruction *sinkThroughTriviallyReplaceablePHI(
11961317 PHINode *TPN, Instruction *I, LoopInfo *LI,
11971318 SmallDenseMap &SunkCopies,
1198 const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop) {
1319 const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
1320 MemorySSAUpdater *MSSAU) {
11991321 assert(isTriviallyReplaceablePHI(*TPN, *I) &&
12001322 "Expect only trivially replaceable PHI");
12011323 BasicBlock *ExitBlock = TPN->getParent();
12041326 if (It != SunkCopies.end())
12051327 New = It->second;
12061328 else
1207 New = SunkCopies[ExitBlock] =
1208 CloneInstructionInExitBlock(*I, *ExitBlock, *TPN, LI, SafetyInfo);
1329 New = SunkCopies[ExitBlock] = CloneInstructionInExitBlock(
1330 *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
12091331 return New;
12101332 }
12111333
12291351
12301352 static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
12311353 LoopInfo *LI, const Loop *CurLoop,
1232 LoopSafetyInfo *SafetyInfo) {
1354 LoopSafetyInfo *SafetyInfo,
1355 MemorySSAUpdater *MSSAU) {
12331356 #ifndef NDEBUG
12341357 SmallVector ExitBlocks;
12351358 CurLoop->getUniqueExitBlocks(ExitBlocks);
12791402 "Expect all predecessors are in the loop");
12801403 if (PN->getBasicBlockIndex(PredBB) >= 0) {
12811404 BasicBlock *NewPred = SplitBlockPredecessors(
1282 ExitBB, PredBB, ".split.loop.exit", DT, LI, nullptr, true);
1405 ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
12831406 // Since we do not allow splitting EH-block with BlockColors in
12841407 // canSplitPredecessors(), we can simply assign predecessor's color to
12851408 // the new block.
13001423 ///
13011424 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
13021425 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
1303 OptimizationRemarkEmitter *ORE, bool FreeInLoop) {
1426 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE,
1427 bool FreeInLoop) {
13041428 LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
13051429 ORE->emit([&]() {
13061430 return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
13521476
13531477 // Split predecessors of the PHI so that we can make users trivially
13541478 // replaceable.
1355 splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo);
1479 splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);
13561480
13571481 // Should rebuild the iterators, as they may be invalidated by
13581482 // splitPredecessorsOfLoopExit().
13871511 assert(ExitBlockSet.count(PN->getParent()) &&
13881512 "The LCSSA PHI is not in an exit block!");
13891513 // The PHI must be trivially replaceable.
1390 Instruction *New = sinkThroughTriviallyReplaceablePHI(PN, &I, LI, SunkCopies,
1391 SafetyInfo, CurLoop);
1514 Instruction *New = sinkThroughTriviallyReplaceablePHI(
1515 PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
13921516 PN->replaceAllUsesWith(New);
1393 eraseInstruction(*PN, *SafetyInfo, nullptr);
1517 eraseInstruction(*PN, *SafetyInfo, nullptr, nullptr);
13941518 Changed = true;
13951519 }
13961520 return Changed;
14011525 ///
14021526 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
14031527 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1404 OptimizationRemarkEmitter *ORE) {
1528 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE) {
14051529 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I
14061530 << "\n");
14071531 ORE->emit([&]() {
14261550 else
14271551 // Move the new node to the destination block, before its terminator.
14281552 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo);
1553 if (MSSAU) {
1554 // If moving, I just moved a load or store, so update MemorySSA.
1555 MemoryUseOrDef *OldMemAcc = cast_or_null(
1556 MSSAU->getMemorySSA()->getMemoryAccess(&I));
1557 if (OldMemAcc)
1558 MSSAU->moveToPlace(OldMemAcc, Dest, MemorySSA::End);
1559 }
14291560
14301561 // Do not retain debug locations when we are moving instructions to different
14311562 // basic blocks, because we want to avoid jumpy line tables. Calls, however,
18301961
18311962 // If the SSAUpdater didn't use the load in the preheader, just zap it now.
18321963 if (PreheaderLoad->use_empty())
1833 eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST);
1964 eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST, nullptr);
18341965
18351966 return true;
18361967 }
19602091 return false;
19612092 }
19622093
2094 static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
2095 Loop *CurLoop) {
2096 MemoryAccess *Source;
2097 // See declaration of EnableLicmCap for usage details.
2098 if (EnableLicmCap)
2099 Source = MU->getDefiningAccess();
2100 else
2101 Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
2102 return !MSSA->isLiveOnEntryDef(Source) &&
2103 CurLoop->contains(Source->getBlock());
2104 }
2105
19632106 /// Little predicate that returns true if the specified basic block is in
19642107 /// a subloop of the current one, not the current one itself.
19652108 ///
303303 // No need to check for instruction's operands are loop invariant.
304304 assert(L.hasLoopInvariantOperands(I) &&
305305 "Insts in a loop's preheader should have loop invariant operands!");
306 if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, false))
306 if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, nullptr, false))
307307 continue;
308308 if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI))
309309 Changed = true;
11 ; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
22 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require,require,require,require,loop(licm)' < %s -S | FileCheck %s
33 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require,require,require,require,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
4 ; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
45
56 declare i32 @foo() readonly argmemonly nounwind
67 declare i32 @foo2() readonly nounwind
1011 ; CHECK-LABEL: @test
1112 ; CHECK: @foo
1213 ; CHECK-LABEL: loop:
14 ; ALIAS-N2-LABEL: @test
15 ; ALIAS-N2: @foo
16 ; ALIAS-N2-LABEL: loop:
1317 br label %loop
1418
1519 loop:
2327 ; CHECK-LABEL: @test_neg
2428 ; CHECK-LABEL: loop:
2529 ; CHECK: @foo
30 ; ALIAS-N2-LABEL: @test_neg
31 ; ALIAS-N2-LABEL: loop:
32 ; ALIAS-N2: @foo
2633 br label %loop
2734
2835 loop:
3542 ; CHECK-LABEL: @test2
3643 ; CHECK: @bar
3744 ; CHECK-LABEL: loop:
45 ; ALIAS-N2-LABEL: @test2
46 ; ALIAS-N2: @bar
47 ; ALIAS-N2-LABEL: loop:
3848 br label %loop
3949
4050 loop:
4858 ; CHECK-LABEL: @test3
4959 ; CHECK-LABEL: loop:
5060 ; CHECK: @bar
61 ; ALIAS-N2-LABEL: @test3
62 ; ALIAS-N2-LABEL: loop:
63 ; ALIAS-N2: @bar
5164 br label %loop
5265
5366 loop:
6376 ; CHECK-LABEL: @test4
6477 ; CHECK-LABEL: loop:
6578 ; CHECK: @bar
79 ; ALIAS-N2-LABEL: @test4
80 ; ALIAS-N2-LABEL: loop:
81 ; ALIAS-N2: @bar
6682 br label %loop
6783
6884 loop:
7692 ; we clump foo_new with bar.
7793 ; With the N2 Alias analysis diagnostic tool, we are able to hoist the
7894 ; argmemonly bar call out of the loop.
95 ; Using MemorySSA we can also hoist bar.
7996
8097 define void @test5(i32* %loc2, i32* noalias %loc) {
8198 ; ALIAS-N2-LABEL: @test5
102119 ; CHECK: %val = load i32, i32* %loc2
103120 ; CHECK-LABEL: loop:
104121 ; CHECK: @llvm.memcpy
122 ; ALIAS-N2-LABEL: @test6
123 ; ALIAS-N2: %val = load i32, i32* %loc2
124 ; ALIAS-N2-LABEL: loop:
125 ; ALIAS-N2: @llvm.memcpy
105126 br label %loop
106127
107128 loop:
118139 ; CHECK: %val = load i32, i32* %loc2
119140 ; CHECK-LABEL: loop:
120141 ; CHECK: @custom_memcpy
142 ; ALIAS-N2-LABEL: @test7
143 ; ALIAS-N2: %val = load i32, i32* %loc2
144 ; ALIAS-N2-LABEL: loop:
145 ; ALIAS-N2: @custom_memcpy
121146 br label %loop
122147
123148 loop:
0 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
11 ; RUN: opt -aa-pipeline=basic-aa -passes='require,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
2 ; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
23
34 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
45 target triple = "x86_64-unknown-linux-gnu"
0 ; RUN: opt < %s -licm -S | FileCheck %s
11 ; RUN: opt < %s -strip-debug -licm -S | FileCheck %s
2 ; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s --check-prefixes=CHECK,MSSA
23
34 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
45 target triple = "x86_64-unknown-linux-gnu"
1516 ; CHECK-NEXT: [[_TMP2:%.*]] = load i32, i32* @a, align 4
1617 ; CHECK-NEXT: [[_TMP3:%.*]] = load i32, i32* @b, align 4
1718 ; CHECK-NEXT: [[_TMP4:%.*]] = sdiv i32 [[_TMP2]], [[_TMP3]]
19 ; MSSA-NEXT: store i32 [[_TMP4:%.*]], i32* @c, align 4
1820 ; CHECK-NEXT: br label [[BB3:%.*]]
21
1922 br label %bb3
2023
2124 bb3: ; preds = %bb3, %0
0 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
11 ; RUN: opt -aa-pipeline=basic-aa -passes='require,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
2 ; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
3 ; RUN: opt -aa-pipeline=basic-aa -passes='require,loop(simplify-cfg,licm)' -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
24
35 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
46 target triple = "x86_64-unknown-linux-gnu"
0 ; RUN: opt -licm -S < %s | FileCheck %s
1 ; RUN: opt -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
12
23 ; Function Attrs: noinline norecurse nounwind readnone ssp uwtable
34 define zeroext i1 @invariant_denom(double %v) #0 {
0 ; REQUIRES: asserts
11 ; RUN: opt < %s -licm -disable-basicaa -stats -S 2>&1 | grep "1 licm"
2 ; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -disable-basicaa -stats -S 2>&1 | grep "1 licm"
23
34 @"\01L_OBJC_METH_VAR_NAME_" = internal global [4 x i8] c"foo\00", section "__TEXT,__objc_methname,cstring_literals", align 1
45 @"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
0 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
11 ; RUN: opt -aa-pipeline=basic-aa -passes='require,loop(licm)' -S %s | FileCheck %s
2 ; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
23 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
34 target triple = "x86_64-unknown-linux-gnu"
45
33 ; RUN: opt -passes='require,loop(licm)' -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
44 ; RUN: opt -passes='require,loop(licm)' -licm-control-flow-hoisting=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
55 ; RUN: opt -passes='require,loop(licm)' -licm-control-flow-hoisting=0 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
6
7 ; RUN: opt -passes='require,loop(licm)' -licm-control-flow-hoisting=1 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
8 ; Enable run below when adding promotion. e.g. "store i32 %phi, i32* %p" is promoted to phi.lcssa.
9 ; opt -passes='require,loop(licm)' -licm-control-flow-hoisting=0 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
10
611
712 ; CHECK-LABEL: @triangle_phi
813 define void @triangle_phi(i32 %x, i32* %p) {
0 ; RUN: opt -S -licm < %s | FileCheck %s
11 ; RUN: opt -aa-pipeline=basic-aa -passes='require,require,require,require,loop(licm)' -S %s | FileCheck %s
2 ; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
23
34 target datalayout = "E-m:e-p:32:32-i8:8:8-i16:16:16-i64:32:32-f64:32:32-v64:32:32-v128:32:32-a0:0:32-n32"
45
0 ; RUN: opt < %s -licm -S | FileCheck %s
11 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='require,loop(licm)' -S | FileCheck %s
2 ; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
23
34 @X = global i32 0 ; [#uses=1]
45
0 ; RUN: opt < %s -basicaa -licm -S | FileCheck %s
1
2 ; Test moved from sinking.ll, as it tests sinking of a store who alone touches
3 ; a memory location in a loop.
4 ; Store can be sunk out of exit block containing indirectbr instructions after
5 ; D50925. Updated to use an argument instead of undef, due to PR38989.
6 define void @test12(i32* %ptr) {
7 ; CHECK-LABEL: @test12
8 ; CHECK: store
9 ; CHECK-NEXT: br label %lab4
10 br label %lab4
11
12 lab4:
13 br label %lab20
14
15 lab5:
16 br label %lab20
17
18 lab6:
19 br label %lab4
20
21 lab7:
22 br i1 undef, label %lab8, label %lab13
23
24 lab8:
25 br i1 undef, label %lab13, label %lab10
26
27 lab10:
28 br label %lab7
29
30 lab13:
31 ret void
32
33 lab20:
34 br label %lab21
35
36 lab21:
37 ; CHECK: lab21:
38 ; CHECK-NOT: store
39 ; CHECK: br i1 false, label %lab21, label %lab22
40 store i32 36127957, i32* %ptr, align 4
41 br i1 undef, label %lab21, label %lab22
42
43 lab22:
44 ; CHECK: lab22:
45 ; CHECK-NOT: store
46 ; CHECK-NEXT: indirectbr i8* undef
47 indirectbr i8* undef, [label %lab5, label %lab6, label %lab7]
48 }
49
11 ; RUN: opt -S -licm < %s | opt -S -loop-sink | FileCheck %s --check-prefix=CHECK-SINK
22 ; RUN: opt -S < %s -passes='require,loop(licm),loop-sink' \
33 ; RUN: | FileCheck %s --check-prefix=CHECK-SINK
4 ; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s --check-prefix=CHECK-LICM
45
56 ; Original source code:
67 ; int g;
0 ; RUN: opt < %s -basicaa -licm -S | FileCheck %s
11 ; RUN: opt < %s -debugify -basicaa -licm -S | FileCheck %s -check-prefix=DEBUGIFY
2 ; RUN: opt < %s -basicaa -licm -S -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
3
24
35 declare i32 @strlen(i8*) readonly nounwind
46
357359 ret i32 %lcssa
358360 }
359361
360 ; Can't sink stores out of exit blocks containing indirectbr instructions
361 ; because loop simplify does not create dedicated exits for such blocks. Test
362 ; that by sinking the store from lab21 to lab22, but not further.
363 define void @test12() {
364 ; CHECK-LABEL: @test12
365 br label %lab4
366
367 lab4:
368 br label %lab20
369
370 lab5:
371 br label %lab20
372
373 lab6:
374 br label %lab4
375
376 lab7:
377 br i1 undef, label %lab8, label %lab13
378
379 lab8:
380 br i1 undef, label %lab13, label %lab10
381
382 lab10:
383 br label %lab7
384
385 lab13:
386 ret void
387
388 lab20:
389 br label %lab21
390
391 lab21:
392 ; CHECK: lab21:
393 ; CHECK-NOT: store
394 ; CHECK: br i1 false, label %lab21, label %lab22
395 store i32 36127957, i32* undef, align 4
396 br i1 undef, label %lab21, label %lab22
397
398 lab22:
399 ; CHECK: lab22:
400 ; CHECK: store
401 ; CHECK-NEXT: indirectbr i8* undef
402 indirectbr i8* undef, [label %lab5, label %lab6, label %lab7]
403 }
362 ; @test12 moved to sink-promote.ll, as it tests sinking and promotion.
404363
405364 ; Test that we don't crash when trying to sink stores and there's no preheader
406365 ; available (which is used for creating loads that may be used by the SSA
0 ; RUN: opt -basicaa -sroa -loop-rotate -licm -S < %s | FileCheck %s
11 ; RUN: opt -basicaa -sroa -loop-rotate %s | opt -aa-pipeline=basic-aa -passes='require,require,require,require,loop(licm)' -S | FileCheck %s
2 ; RUN: opt -basicaa -sroa -loop-rotate -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
23 ; The objects *p and *q are aliased to each other, but even though *q is
34 ; volatile, *p can be considered invariant in the loop. Check if it is moved
45 ; out of the loop.