llvm.org GIT mirror llvm / 5927508
[LICM] Use ICFLoopSafetyInfo in LICM This patch makes LICM use `ICFLoopSafetyInfo` that is a smarter version of LoopSafetyInfo that leverages power of Implicit Control Flow Tracking to keep track of throwing instructions and give less pessimistic answers to queries related to throws. The ICFLoopSafetyInfo itself has been introduced in rL344601. This patch enables it in LICM only. Differential Revision: https://reviews.llvm.org/D50377 Reviewed By: apilipenko git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@346201 91177308-0d34-0410-b5e6-96231b3b80d8 Max Kazantsev 11 months ago
7 changed file(s) with 240 addition(s) and 33 deletion(s). Raw diff Collapse all Expand all
108108 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
109109 bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
110110 TargetLibraryInfo *, TargetTransformInfo *, Loop *,
111 AliasSetTracker *, LoopSafetyInfo *,
111 AliasSetTracker *, ICFLoopSafetyInfo *,
112112 OptimizationRemarkEmitter *ORE);
113113
114114 /// Walk the specified region of the CFG (defined by all blocks
121121 /// ORE. It returns changed status.
122122 bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
123123 TargetLibraryInfo *, Loop *, AliasSetTracker *,
124 LoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
124 ICFLoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
125125
126126 /// This function deletes dead loops. The caller of this function needs to
127127 /// guarantee that the loop is infact dead.
150150 SmallVectorImpl &,
151151 PredIteratorCache &, LoopInfo *,
152152 DominatorTree *, const TargetLibraryInfo *,
153 Loop *, AliasSetTracker *, LoopSafetyInfo *,
153 Loop *, AliasSetTracker *,
154 ICFLoopSafetyInfo *,
154155 OptimizationRemarkEmitter *);
155156
156157 /// Does a BFS from a given node to all of its children inside a given loop.
102102 const LoopSafetyInfo *SafetyInfo,
103103 TargetTransformInfo *TTI, bool &FreeInLoop);
104104 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
105 LoopSafetyInfo *SafetyInfo,
105 ICFLoopSafetyInfo *SafetyInfo,
106106 OptimizationRemarkEmitter *ORE);
107107 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
108 const Loop *CurLoop, LoopSafetyInfo *SafetyInfo,
108 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
109109 OptimizationRemarkEmitter *ORE, bool FreeInLoop);
110110 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
111111 const DominatorTree *DT,
122122 const LoopInfo *LI,
123123 const LoopSafetyInfo *SafetyInfo);
124124
125 static void eraseInstruction(Instruction &I, AliasSetTracker *AST);
125 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
126 AliasSetTracker *AST);
126127
127128 namespace {
128129 struct LoopInvariantCodeMotion {
268269 BasicBlock *Preheader = L->getLoopPreheader();
269270
270271 // Compute loop safety information.
271 SimpleLoopSafetyInfo SafetyInfo;
272 ICFLoopSafetyInfo SafetyInfo(DT);
272273 SafetyInfo.computeLoopSafetyInfo(L);
273274
274275 // We want to visit all of the instructions in this loop... that are not parts
375376 bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
376377 DominatorTree *DT, TargetLibraryInfo *TLI,
377378 TargetTransformInfo *TTI, Loop *CurLoop,
378 AliasSetTracker *CurAST, LoopSafetyInfo *SafetyInfo,
379 AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
379380 OptimizationRemarkEmitter *ORE) {
380381
381382 // Verify inputs.
405406 LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n');
406407 salvageDebugInfo(I);
407408 ++II;
408 eraseInstruction(I, CurAST);
409 eraseInstruction(I, *SafetyInfo, CurAST);
409410 Changed = true;
410411 continue;
411412 }
422423 if (sink(I, LI, DT, CurLoop, SafetyInfo, ORE, FreeInLoop)) {
423424 if (!FreeInLoop) {
424425 ++II;
425 eraseInstruction(I, CurAST);
426 eraseInstruction(I, *SafetyInfo, CurAST);
426427 }
427428 Changed = true;
428429 }
439440 ///
440441 bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
441442 DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
442 AliasSetTracker *CurAST, LoopSafetyInfo *SafetyInfo,
443 AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
443444 OptimizationRemarkEmitter *ORE) {
444445 // Verify inputs.
445446 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
480481 CurAST->copyValue(&I, C);
481482 I.replaceAllUsesWith(C);
482483 if (isInstructionTriviallyDead(&I, TLI))
483 eraseInstruction(I, CurAST);
484 eraseInstruction(I, *SafetyInfo, CurAST);
484485 Changed = true;
485486 continue;
486487 }
509510 auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
510511 auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
511512 ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
513 SafetyInfo->insertInstructionTo(I.getParent());
512514 ReciprocalDivisor->insertBefore(&I);
513515
514516 auto Product =
515517 BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
516518 Product->setFastMathFlags(I.getFastMathFlags());
519 SafetyInfo->insertInstructionTo(I.getParent());
517520 Product->insertAfter(&I);
518521 I.replaceAllUsesWith(Product);
519 eraseInstruction(I, CurAST);
522 eraseInstruction(I, *SafetyInfo, CurAST);
520523
521524 hoist(*ReciprocalDivisor, DT, CurLoop, SafetyInfo, ORE);
522525 Changed = true;
885888 return New;
886889 }
887890
888 static void eraseInstruction(Instruction &I, AliasSetTracker *AST) {
891 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
892 AliasSetTracker *AST) {
889893 if (AST)
890894 AST->deleteValue(&I);
895 SafetyInfo.removeInstruction(&I);
891896 I.eraseFromParent();
892897 }
893898
9981003 /// position, and may either delete it or move it to outside of the loop.
9991004 ///
10001005 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
1001 const Loop *CurLoop, LoopSafetyInfo *SafetyInfo,
1006 const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
10021007 OptimizationRemarkEmitter *ORE, bool FreeInLoop) {
10031008 LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
10041009 ORE->emit([&]() {
10891094 Instruction *New = sinkThroughTriviallyReplaceablePHI(PN, &I, LI, SunkCopies,
10901095 SafetyInfo, CurLoop);
10911096 PN->replaceAllUsesWith(New);
1092 eraseInstruction(*PN, nullptr);
1097 eraseInstruction(*PN, *SafetyInfo, nullptr);
10931098 Changed = true;
10941099 }
10951100 return Changed;
10991104 /// is safe to hoist, this instruction is called to do the dirty work.
11001105 ///
11011106 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1102 LoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
1107 ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
11031108 auto *Preheader = CurLoop->getLoopPreheader();
11041109 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Preheader->getName() << ": " << I
11051110 << "\n");
11191124 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
11201125 I.dropUnknownNonDebugMetadata();
11211126
1127 SafetyInfo->removeInstruction(&I);
1128 SafetyInfo->insertInstructionTo(Preheader);
11221129 // Move the new node to the Preheader, before its terminator.
11231130 I.moveBefore(Preheader->getTerminator());
11241131
11791186 int Alignment;
11801187 bool UnorderedAtomic;
11811188 AAMDNodes AATags;
1189 ICFLoopSafetyInfo &SafetyInfo;
11821190
11831191 Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
11841192 if (Instruction *I = dyn_cast(V))
12011209 SmallVectorImpl &LEB,
12021210 SmallVectorImpl &LIP, PredIteratorCache &PIC,
12031211 AliasSetTracker &ast, LoopInfo &li, DebugLoc dl, int alignment,
1204 bool UnorderedAtomic, const AAMDNodes &AATags)
1212 bool UnorderedAtomic, const AAMDNodes &AATags,
1213 ICFLoopSafetyInfo &SafetyInfo)
12051214 : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
12061215 LoopExitBlocks(LEB), LoopInsertPts(LIP), PredCache(PIC), AST(ast),
12071216 LI(li), DL(std::move(dl)), Alignment(alignment),
1208 UnorderedAtomic(UnorderedAtomic), AATags(AATags) {}
1217 UnorderedAtomic(UnorderedAtomic), AATags(AATags), SafetyInfo(SafetyInfo)
1218 {}
12091219
12101220 bool isInstInList(Instruction *I,
12111221 const SmallVectorImpl &) const override {
12421252 // Update alias analysis.
12431253 AST.copyValue(LI, V);
12441254 }
1245 void instructionDeleted(Instruction *I) const override { AST.deleteValue(I); }
1255 void instructionDeleted(Instruction *I) const override {
1256 SafetyInfo.removeInstruction(I);
1257 AST.deleteValue(I);
1258 }
12461259 };
12471260
12481261
12801293 SmallVectorImpl &ExitBlocks,
12811294 SmallVectorImpl &InsertPts, PredIteratorCache &PIC,
12821295 LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
1283 Loop *CurLoop, AliasSetTracker *CurAST, LoopSafetyInfo *SafetyInfo,
1296 Loop *CurLoop, AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
12841297 OptimizationRemarkEmitter *ORE) {
12851298 // Verify inputs.
12861299 assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&
14991512 SSAUpdater SSA(&NewPHIs);
15001513 LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
15011514 InsertPts, PIC, *CurAST, *LI, DL, Alignment,
1502 SawUnorderedAtomic, AATags);
1515 SawUnorderedAtomic, AATags, *SafetyInfo);
15031516
15041517 // Set up the preheader to have a definition of the value. It is the live-out
15051518 // value from the preheader that uses in the loop will use.
15191532
15201533 // If the SSAUpdater didn't use the load in the preheader, just zap it now.
15211534 if (PreheaderLoad->use_empty())
1522 eraseInstruction(*PreheaderLoad, CurAST);
1535 eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST);
15231536
15241537 return true;
15251538 }
1717 %tmp4 = bitcast half %tmp3 to i16
1818 %tmp5 = insertelement <2 x i16> , i16 %tmp4, i32 1
1919 %tmp6 = bitcast i16* %arg to half*
20 store half %tmp2, half* %tmp6, align 2
20 store volatile half %tmp2, half* %tmp6, align 2
2121 %tmp7 = bitcast <2 x i16> %tmp to <2 x half>
2222 %tmp8 = extractelement <2 x half> %tmp7, i32 0
2323 br label %bb1
8484 }
8585
8686
87 ; TODO: We can also hoist this load and guard from mustexec non-header block.
87 ; TODO: We can also hoist this guard from mustexec non-header block.
8888 define void @test4(i1 %c, i32* %p) {
8989
9090 ; CHECK-LABEL: @test4(
9191 ; CHECK-LABEL: entry:
92 ; CHECK-LABEL: loop:
93 ; CHECK-LABEL: backedge:
9492 ; CHECK: %a = load i32, i32* %p
9593 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
94 ; CHECK-LABEL: loop:
95 ; CHECK-LABEL: backedge:
9696 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
9797
9898 entry:
455455 exit:
456456 ret void
457457 }
458
459 ; Check that we can hoist a mustexecute load from backedge even if something
460 ; throws after it.
461 define void @test_hoist_from_backedge_01(i32* %p, i32 %n) {
462
463 ; CHECK-LABEL: @test_hoist_from_backedge_01(
464 ; CHECK: entry:
465 ; CHECK-NEXT: %load = load i32, i32* %p
466 ; CHECK-NOT: load i32
467
468 entry:
469 br label %loop
470
471 loop:
472 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
473 %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
474 %cond = icmp slt i32 %iv, %n
475 br i1 %cond, label %if.true, label %if.false
476
477 if.true:
478 %a = add i32 %iv, %iv
479 br label %backedge
480
481 if.false:
482 %b = mul i32 %iv, %iv
483 br label %backedge
484
485 backedge:
486 %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
487 %iv.next = add i32 %iv, %merge
488 %load = load i32, i32* %p
489 call void @may_throw()
490 %loop.cond = icmp ult i32 %iv.next, %load
491 br i1 %loop.cond, label %loop, label %exit
492
493 exit:
494 ret void
495 }
496
497 ; Check that we don't hoist the load if something before it can throw.
498 define void @test_hoist_from_backedge_02(i32* %p, i32 %n) {
499
500 ; CHECK-LABEL: @test_hoist_from_backedge_02(
501 ; CHECK: entry:
502 ; CHECK: loop:
503 ; CHECK: %load = load i32, i32* %p
504
505 entry:
506 br label %loop
507
508 loop:
509 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
510 %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
511 %cond = icmp slt i32 %iv, %n
512 br i1 %cond, label %if.true, label %if.false
513
514 if.true:
515 %a = add i32 %iv, %iv
516 br label %backedge
517
518 if.false:
519 %b = mul i32 %iv, %iv
520 br label %backedge
521
522 backedge:
523 %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
524 %iv.next = add i32 %iv, %merge
525 call void @may_throw()
526 %load = load i32, i32* %p
527 %loop.cond = icmp ult i32 %iv.next, %load
528 br i1 %loop.cond, label %loop, label %exit
529
530 exit:
531 ret void
532 }
533
534 define void @test_hoist_from_backedge_03(i32* %p, i32 %n) {
535
536 ; CHECK-LABEL: @test_hoist_from_backedge_03(
537 ; CHECK: entry:
538 ; CHECK: loop:
539 ; CHECK: %load = load i32, i32* %p
540
541 entry:
542 br label %loop
543
544 loop:
545 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
546 %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
547 %cond = icmp slt i32 %iv, %n
548 br i1 %cond, label %if.true, label %if.false
549
550 if.true:
551 %a = add i32 %iv, %iv
552 br label %backedge
553
554 if.false:
555 %b = mul i32 %iv, %iv
556 call void @may_throw()
557 br label %backedge
558
559 backedge:
560 %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
561 %iv.next = add i32 %iv, %merge
562 %load = load i32, i32* %p
563 %loop.cond = icmp ult i32 %iv.next, %load
564 br i1 %loop.cond, label %loop, label %exit
565
566 exit:
567 ret void
568 }
569
570 define void @test_hoist_from_backedge_04(i32* %p, i32 %n) {
571
572 ; CHECK-LABEL: @test_hoist_from_backedge_04(
573 ; CHECK: entry:
574 ; CHECK: loop:
575 ; CHECK: %load = load i32, i32* %p
576
577 entry:
578 br label %loop
579
580 loop:
581 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
582 %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
583 call void @may_throw()
584 %cond = icmp slt i32 %iv, %n
585 br i1 %cond, label %if.true, label %if.false
586
587 if.true:
588 %a = add i32 %iv, %iv
589 br label %backedge
590
591 if.false:
592 %b = mul i32 %iv, %iv
593 br label %backedge
594
595 backedge:
596 %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
597 %iv.next = add i32 %iv, %merge
598 %load = load i32, i32* %p
599 %loop.cond = icmp ult i32 %iv.next, %load
600 br i1 %loop.cond, label %loop, label %exit
601
602 exit:
603 ret void
604 }
4848 ret i32 0
4949 }
5050
51 ; Don't hoist load past volatile load.
51 ; Hoist a non-volatile load past volatile load.
5252 define i32 @test3(i32* noalias nocapture readonly %a, i32* %v) nounwind uwtable {
5353 ; CHECK-LABEL: @test3(
5454 entry:
5555 br label %for.body
5656
57 ; CHECK: load i32
58 ; CHECK: for.body:
5759 ; CHECK: load volatile i32
58 ; CHECK-NEXT: load i32
60 ; CHECK-NOT: load
5961 for.body:
6062 %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
6163 %x.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
6971 for.cond.cleanup:
7072 ret i32 %add
7173 }
74
75 ; Don't a volatile load past volatile load.
76 define i32 @test4(i32* noalias nocapture readonly %a, i32* %v) nounwind uwtable {
77 ; CHECK-LABEL: @test4(
78 entry:
79 br label %for.body
80
81 ; CHECK: for.body:
82 ; CHECK: load volatile i32
83 ; CHECK-NEXT: load volatile i32
84 for.body:
85 %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
86 %x.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
87 %xxx = load volatile i32, i32* %v, align 4
88 %i1 = load volatile i32, i32* %a, align 4
89 %add = add nsw i32 %i1, %x.05
90 %inc = add nuw nsw i32 %i.06, 1
91 %exitcond = icmp eq i32 %inc, 1000
92 br i1 %exitcond, label %for.cond.cleanup, label %for.body
93
94 for.cond.cleanup:
95 ret i32 %add
96 }
111111 exit:
112112 ret void
113113 }
114 ; Negative test - can't move out of throwing block
115 define void @nothrow_header_neg(i64 %x, i64 %y, i1 %cond) {
116 ; CHECK-LABEL: nothrow_header_neg
114
115 ; Positive test - can hoist something that happens before thrower.
116 define void @nothrow_header_pos(i64 %x, i64 %y, i1 %cond) {
117 ; CHECK-LABEL: nothrow_header_pos
117118 ; CHECK-LABEL: entry
119 ; CHECK: %div = udiv i64 %x, %y
118120 ; CHECK-LABEL: loop
119 ; CHECK: %div = udiv i64 %x, %y
120121 ; CHECK: call void @use(i64 %div)
121122 entry:
122123 br label %loop
127128 call void @use(i64 %div)
128129 br label %loop
129130 }
131
132
133 ; Negative test - can't move out of throwing block
134 define void @nothrow_header_neg(i64 %x, i64 %y, i1 %cond) {
135 ; CHECK-LABEL: nothrow_header_neg
136 ; CHECK-LABEL: entry
137 ; CHECK-LABEL: loop
138 ; CHECK: call void @maythrow()
139 ; CHECK: %div = udiv i64 %x, %y
140 ; CHECK: call void @use(i64 %div)
141 entry:
142 br label %loop
143 loop: ; preds = %entry, %for.inc
144 br label %loop-if
145 loop-if:
146 call void @maythrow()
147 %div = udiv i64 %x, %y
148 call void @use(i64 %div)
149 br label %loop
150 }