llvm.org GIT mirror llvm / c66d417
[LoopUnrollRuntime] NFC: Refactored safety checks of unrolling multi-exit loop Refactored the code and separated out a function `canSafelyUnrollMultiExitLoop` to reduce redundant checks and make it easier to add profitability heuristics later. Added tests to runtime unrolling to make sure that unrolling for multi-exit loops is not done unless the option -unroll-runtime-multi-exit is true. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@307843 91177308-0d34-0410-b5e6-96231b3b80d8 Anna Thomas 2 years ago
2 changed file(s) with 128 addition(s) and 49 deletion(s). Raw diff Collapse all Expand all
425425 else
426426 return nullptr;
427427 }
428
429 /// Returns true if we can safely unroll a multi-exit/exiting loop. OtherExits
430 /// is populated with all the loop exit blocks other than the LatchExit block.
431 static bool
432 canSafelyUnrollMultiExitLoop(Loop *L, SmallVectorImpl &OtherExits,
433 BasicBlock *LatchExit, bool PreserveLCSSA,
434 bool UseEpilogRemainder) {
435
436 // Support runtime unrolling for multiple exit blocks and multiple exiting
437 // blocks.
438 if (!UnrollRuntimeMultiExit)
439 return false;
440 // Even if runtime multi exit is enabled, we currently have some correctness
441 // constrains in unrolling a multi-exit loop.
442 // We rely on LCSSA form being preserved when the exit blocks are transformed.
443 if (!PreserveLCSSA)
444 return false;
445 SmallVector Exits;
446 L->getUniqueExitBlocks(Exits);
447 for (auto *BB : Exits)
448 if (BB != LatchExit)
449 OtherExits.push_back(BB);
450
451 // TODO: Support multiple exiting blocks jumping to the `LatchExit` when
452 // UnrollRuntimeMultiExit is true. This will need updating the logic in
453 // connectEpilog/connectProlog.
454 if (!LatchExit->getSinglePredecessor()) {
455 DEBUG(dbgs() << "Bailout for multi-exit handling when latch exit has >1 "
456 "predecessor.\n");
457 return false;
458 }
459 // FIXME: We bail out of multi-exit unrolling when epilog loop is generated
460 // and L is an inner loop. This is because in presence of multiple exits, the
461 // outer loop is incorrect: we do not add the EpilogPreheader and exit to the
462 // outer loop. This is automatically handled in the prolog case, so we do not
463 // have that bug in prolog generation.
464 if (UseEpilogRemainder && L->getParentLoop())
465 return false;
466
467 // All constraints have been satisfied.
468 return true;
469 }
470
471
428472
429473 /// Insert code in the prolog/epilog code when unrolling a loop with a
430474 /// run-time trip-count.
469513 bool UseEpilogRemainder,
470514 LoopInfo *LI, ScalarEvolution *SE,
471515 DominatorTree *DT, bool PreserveLCSSA) {
472 bool hasMultipleExitingBlocks = !L->getExitingBlock();
473516 DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
474517 DEBUG(L->dump());
475 // Support only single exiting block unless UnrollRuntimeMultiExit is true.
476 if (!UnrollRuntimeMultiExit && hasMultipleExitingBlocks) {
477 DEBUG(
478 dbgs()
479 << "Multiple exiting blocks and UnrollRuntimeMultiExit not enabled!\n");
480 return false;
481 }
482518
483519 // Make sure the loop is in canonical form.
484520 if (!L->isLoopSimplifyForm()) {
490526 BasicBlock *Latch = L->getLoopLatch();
491527 BasicBlock *Header = L->getHeader();
492528
493 BasicBlock *LatchExit = L->getUniqueExitBlock(); // successor out of loop
494 if (!LatchExit && !UnrollRuntimeMultiExit) {
495 DEBUG(dbgs() << "No unique exit block and UnrollRuntimeMultiExit not enabled\n");
496 return false;
497 }
498 // These are exit blocks other than the target of the latch exiting block.
499 SmallVector OtherExits;
500529 BranchInst *LatchBR = cast(Latch->getTerminator());
501 unsigned int ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0;
530 unsigned ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0;
531 BasicBlock *LatchExit = LatchBR->getSuccessor(ExitIndex);
502532 // Cloning the loop basic blocks (`CloneLoopBlocks`) requires that one of the
503533 // targets of the Latch be an exit block out of the loop. This needs
504534 // to be guaranteed by the callers of UnrollRuntimeLoopRemainder.
505 assert(!L->contains(LatchBR->getSuccessor(ExitIndex)) &&
535 assert(!L->contains(LatchExit) &&
506536 "one of the loop latch successors should be the exit block!");
507 // Support runtime unrolling for multiple exit blocks and multiple exiting
508 // blocks.
509 if (!LatchExit) {
510 LatchExit = LatchBR->getSuccessor(ExitIndex);
511 // We rely on LCSSA form being preserved when the exit blocks are
512 // transformed.
513 if (!PreserveLCSSA)
514 return false;
515 SmallVector Exits;
516 L->getUniqueExitBlocks(Exits);
517 for (auto *BB : Exits)
518 if (BB != LatchExit)
519 OtherExits.push_back(BB);
520 }
521
522 assert(LatchExit && "Latch Exit should exist!");
523
524 // TODO: Support multiple exiting blocks jumping to the `LatchExit` when
525 // UnrollRuntimeMultiExit is true. This will need updating the logic in
526 // connectEpilog.
527 if (!LatchExit->getSinglePredecessor()) {
528 DEBUG(dbgs() << "Bailout for multi-exit handling when latch exit has >1 "
529 "predecessor.\n");
530 return false;
531 }
532 // FIXME: We bail out of multi-exit unrolling when epilog loop is generated
533 // and L is an inner loop. This is because in presence of multiple exits, the
534 // outer loop is incorrect: we do not add the EpilogPreheader and exit to the
535 // outer loop. This is automatically handled in the prolog case, so we do not
536 // have that bug in prolog generation.
537 if (hasMultipleExitingBlocks && UseEpilogRemainder && L->getParentLoop())
538 return false;
537 // These are exit blocks other than the target of the latch exiting block.
538 SmallVector OtherExits;
539 bool isMultiExitUnrollingEnabled = canSafelyUnrollMultiExitLoop(
540 L, OtherExits, LatchExit, PreserveLCSSA, UseEpilogRemainder);
541 // Support only single exit and exiting block unless multi-exit loop unrolling is enabled.
542 if (!isMultiExitUnrollingEnabled &&
543 (!L->getExitingBlock() || OtherExits.size())) {
544 DEBUG(
545 dbgs()
546 << "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
547 "enabled!\n");
548 return false;
549 }
539550 // Use Scalar Evolution to compute the trip count. This allows more loops to
540551 // be unrolled than relying on induction var simplification.
541552 if (!SE)
169169 ret i16 %res.0.lcssa
170170 }
171171
172 ; dont unroll loop with multiple exit/exiting blocks, unless
173 ; -runtime-unroll-multi-exit=true
174 ; single exit, multiple exiting blocks.
175 define void @unique_exit(i32 %arg) {
176 ; PROLOG: unique_exit(
177 ; PROLOG-NOT: .unr
178
179 ; EPILOG: unique_exit(
180 ; EPILOG-NOT: .unr
181 entry:
182 %tmp = icmp sgt i32 undef, %arg
183 br i1 %tmp, label %preheader, label %returnblock
184
185 preheader: ; preds = %entry
186 br label %header
187
188 LoopExit: ; preds = %header, %latch
189 %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
190 br label %returnblock
191
192 returnblock: ; preds = %LoopExit, %entry
193 %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %LoopExit ]
194 ret void
195
196 header: ; preds = %preheader, %latch
197 %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
198 %inc = add nsw i32 %tmp4, 1
199 br i1 true, label %LoopExit, label %latch
200
201 latch: ; preds = %header
202 %cmp = icmp slt i32 %inc, undef
203 br i1 %cmp, label %header, label %LoopExit
204 }
205
206 ; multiple exit blocks. don't unroll
207 define void @multi_exit(i64 %trip, i1 %cond) {
208 ; PROLOG: multi_exit(
209 ; PROLOG-NOT: .unr
210
211 ; EPILOG: multi_exit(
212 ; EPILOG-NOT: .unr
213 entry:
214 br label %loop_header
215
216 loop_header:
217 %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
218 br i1 %cond, label %loop_latch, label %loop_exiting_bb1
219
220 loop_exiting_bb1:
221 br i1 false, label %loop_exiting_bb2, label %exit1
222
223 loop_exiting_bb2:
224 br i1 false, label %loop_latch, label %exit3
225
226 exit3:
227 ret void
228
229 loop_latch:
230 %iv_next = add i64 %iv, 1
231 %cmp = icmp ne i64 %iv_next, %trip
232 br i1 %cmp, label %loop_header, label %exit2.loopexit
233
234 exit1:
235 ret void
236
237 exit2.loopexit:
238 ret void
239 }
172240 !0 = distinct !{!0, !1}
173241 !1 = !{!"llvm.loop.unroll.runtime.disable"}
174242