llvm.org GIT mirror llvm / bd436d0
[Loop Peeling] Enable peeling for loops with multiple exits This CL enables peeling of the loop with multiple exits where one exit should be from latch and others are basic blocks with call to deopt. The peeling is enabled under the flag which is false by default. Reviewers: reames, mkuper, iajbar, fhahn Reviewed By: reames Subscribers: xbolva00, hiraditya, zzheng, llvm-commits Differential Revision: https://reviews.llvm.org/D63923 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@366048 91177308-0d34-0410-b5e6-96231b3b80d8 Serguei Katkov a month ago
3 changed file(s) with 102 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
401401 // counts. If we later unroll the loop, we want these to be updated.
402402 if (Peeled) {
403403 // According to our guards and profitability checks the only
404 // meaningful exit should be latch block.
404 // meaningful exit should be latch block. Other exits go to deopt,
405 // so we do not worry about them.
405406 BasicBlock *ExitingBlock = L->getLoopLatch();
406407 assert(ExitingBlock && "Loop without exiting block?");
407408 assert(L->isLoopExiting(ExitingBlock) && "Latch is not exiting?");
6060 "unroll-force-peel-count", cl::init(0), cl::Hidden,
6161 cl::desc("Force a peel count regardless of profiling information."));
6262
63 static cl::opt UnrollPeelMultiDeoptExit(
64 "unroll-peel-multi-deopt-exit", cl::init(false), cl::Hidden,
65 cl::desc("Allow peeling of loops with multiple deopt exits."));
66
6367 // Designates that a Phi is estimated to become invariant after an "infinite"
6468 // number of loop iterations (i.e. only may become an invariant if the loop is
6569 // fully unrolled).
7175 // Make sure the loop is in simplified form
7276 if (!L->isLoopSimplifyForm())
7377 return false;
78
79 if (UnrollPeelMultiDeoptExit) {
80 SmallVector Exits;
81 L->getUniqueNonLatchExitBlocks(Exits);
82
83 if (!Exits.empty()) {
84 // Latch's terminator is a conditional branch, Latch is exiting and
85 // all non Latch exits ends up with deoptimize.
86 const BasicBlock *Latch = L->getLoopLatch();
87 const BranchInst *T = dyn_cast(Latch->getTerminator());
88 return T && T->isConditional() && L->isLoopExiting(Latch) &&
89 all_of(Exits, [](const BasicBlock *BB) {
90 return BB->getTerminatingDeoptimizeCall();
91 });
92 }
93 }
7494
7595 // Only peel loops that contain a single exit
7696 if (!L->getExitingBlock() || !L->getUniqueExitBlock())
0 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll -unroll-runtime -unroll-peel-multi-deopt-exit 2>&1 | FileCheck %s
1 ; RUN: opt < %s -S -debug-only=loop-unroll -unroll-peel-multi-deopt-exit -passes='require,function(require,unroll)' 2>&1 | FileCheck %s
2
3 ; Make sure we use the profile information correctly to peel-off 3 iterations
4 ; from the loop, and update the branch weights for the peeled loop properly.
5
6 ; CHECK: Loop Unroll: F[basic]
7 ; CHECK: PEELING loop %for.body with iteration count 3!
8
9 ; CHECK-LABEL: @basic
10 ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !16
11 ; CHECK: [[NEXT0]]:
12 ; CHECK: br i1 %{{.*}}, label %[[NEXT1:.*]], label %for.cond.for.end_crit_edge, !prof !17
13 ; CHECK: [[NEXT1]]:
14 ; CHECK: br i1 %{{.*}}, label %[[NEXT2:.*]], label %for.cond.for.end_crit_edge, !prof !18
15 ; CHECK: [[NEXT2]]:
16 ; CHECK: br i1 %{{.*}}, label %for.body, label %{{.*}}, !prof !19
17
18 define i32 @basic(i32* %p, i32 %k, i1 %c) #0 !prof !15 {
19 entry:
20 %cmp3 = icmp slt i32 0, %k
21 br i1 %cmp3, label %for.body.lr.ph, label %for.end
22
23 for.body.lr.ph: ; preds = %entry
24 br label %for.body
25
26 for.body: ; preds = %for.body.lr.ph, %for.body
27 %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %continue ]
28 %p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %continue ]
29 %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
30 store i32 %i.05, i32* %p.addr.04, align 4
31 %inc = add nsw i32 %i.05, 1
32 %cmp = icmp slt i32 %inc, %k
33 br i1 %c, label %continue, label %side_exit, !prof !17
34
35 continue:
36 br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !16
37
38 for.cond.for.end_crit_edge: ; preds = %for.body
39 br label %for.end
40
41 for.end: ; preds = %for.cond.for.end_crit_edge, %entry
42 %res = phi i32 [ 0, %entry ], [ %inc, %for.cond.for.end_crit_edge ]
43 ret i32 %res
44
45 side_exit:
46 %rval = call i32(...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %inc) ]
47 ret i32 %rval
48 }
49
50 declare i32 @llvm.experimental.deoptimize.i32(...)
51
52 attributes #0 = { nounwind }
53 attributes #1 = { nounwind optsize }
54
55 !llvm.module.flags = !{!1}
56
57 !1 = !{i32 1, !"ProfileSummary", !2}
58 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
59 !3 = !{!"ProfileFormat", !"InstrProf"}
60 !4 = !{!"TotalCount", i64 10}
61 !5 = !{!"MaxCount", i64 3}
62 !6 = !{!"MaxInternalCount", i64 1}
63 !7 = !{!"MaxFunctionCount", i64 3}
64 !8 = !{!"NumCounts", i64 2}
65 !9 = !{!"NumFunctions", i64 2}
66 !10 = !{!"DetailedSummary", !11}
67 !11 = !{!12, !13, !14}
68 !12 = !{i32 10000, i64 3, i32 2}
69 !13 = !{i32 999000, i64 1, i32 10}
70 !14 = !{i32 999999, i64 1, i32 10}
71 !15 = !{!"function_entry_count", i64 1}
72 !16 = !{!"branch_weights", i32 3001, i32 1001}
73 !17 = !{!"branch_weights", i32 1, i32 0}
74
75 ;CHECK: !16 = !{!"branch_weights", i32 900, i32 101}
76 ;CHECK: !17 = !{!"branch_weights", i32 540, i32 360}
77 ;CHECK: !18 = !{!"branch_weights", i32 162, i32 378}
78 ;CHECK: !19 = !{!"branch_weights", i32 1399, i32 162}
79