llvm.org GIT mirror llvm / 417182e
Backport of rL326666 and rL326668 for PR36607 and PR36608. [CallSiteSplitting] properly split musttail calls. The original author was Fedor Indutny <fedor@indutny.com>. `musttail` calls can't be naively splitted. The split blocks must include not only the call instruction itself, but also (optional) `bitcast` and `return` instructions that follow it. Clone `bitcast` and `ret`, place them into the split blocks, and remove the tail block when done. Reviewers: junbuml, mcrosier, davidxl, davide, fhahn Reviewed By: fhahn Subscribers: JDevlieghere, llvm-commits Differential Revision: https://reviews.llvm.org/D43729 git-svn-id: https://llvm.org/svn/llvm-project/llvm/branches/release_60@329793 91177308-0d34-0410-b5e6-96231b3b80d8 Florian Hahn 1 year, 5 months ago
2 changed file(s) with 184 addition(s) and 2 deletion(s). Raw diff Collapse all Expand all
200200 return CallSiteBB->canSplitPredecessors();
201201 }
202202
203 static Instruction *cloneInstForMustTail(Instruction *I, Instruction *Before,
204 Value *V) {
205 Instruction *Copy = I->clone();
206 Copy->setName(I->getName());
207 Copy->insertBefore(Before);
208 if (V)
209 Copy->setOperand(0, V);
210 return Copy;
211 }
212
213 /// Copy mandatory `musttail` return sequence that follows original `CI`, and
214 /// link it up to `NewCI` value instead:
215 ///
216 /// * (optional) `bitcast NewCI to ...`
217 /// * `ret bitcast or NewCI`
218 ///
219 /// Insert this sequence right before `SplitBB`'s terminator, which will be
220 /// cleaned up later in `splitCallSite` below.
221 static void copyMustTailReturn(BasicBlock *SplitBB, Instruction *CI,
222 Instruction *NewCI) {
223 bool IsVoid = SplitBB->getParent()->getReturnType()->isVoidTy();
224 auto II = std::next(CI->getIterator());
225
226 BitCastInst *BCI = dyn_cast(&*II);
227 if (BCI)
228 ++II;
229
230 ReturnInst *RI = dyn_cast(&*II);
231 assert(RI && "`musttail` call must be followed by `ret` instruction");
232
233 TerminatorInst *TI = SplitBB->getTerminator();
234 Value *V = NewCI;
235 if (BCI)
236 V = cloneInstForMustTail(BCI, TI, V);
237 cloneInstForMustTail(RI, TI, IsVoid ? nullptr : V);
238
239 // FIXME: remove TI here, `DuplicateInstructionsInSplitBetween` has a bug
240 // that prevents doing this now.
241 }
242
203243 /// Return true if the CS is split into its new predecessors which are directly
204244 /// hooked to each of its original predecessors pointed by PredBB1 and PredBB2.
205245 /// CallInst1 and CallInst2 will be the new call-sites placed in the new
244284 Instruction *CallInst1, Instruction *CallInst2) {
245285 Instruction *Instr = CS.getInstruction();
246286 BasicBlock *TailBB = Instr->getParent();
287 bool IsMustTailCall = CS.isMustTailCall();
247288 assert(Instr == (TailBB->getFirstNonPHIOrDbg()) && "Unexpected call-site");
248289
249290 BasicBlock *SplitBlock1 =
275316 ++ArgNo;
276317 }
277318 }
319 // Clone and place bitcast and return instructions before `TI`
320 if (IsMustTailCall) {
321 copyMustTailReturn(SplitBlock1, CS.getInstruction(), CallInst1);
322 copyMustTailReturn(SplitBlock2, CS.getInstruction(), CallInst2);
323 }
278324
279325 // Replace users of the original call with a PHI mering call-sites split.
280 if (Instr->getNumUses()) {
326 if (!IsMustTailCall && Instr->getNumUses()) {
281327 PHINode *PN = PHINode::Create(Instr->getType(), 2, "phi.call",
282328 TailBB->getFirstNonPHI());
283329 PN->addIncoming(CallInst1, SplitBlock1);
289335 << "\n");
290336 DEBUG(dbgs() << " " << *CallInst2 << " in " << SplitBlock2->getName()
291337 << "\n");
338
339 NumCallSiteSplit++;
340
341 // FIXME: remove TI in `copyMustTailReturn`
342 if (IsMustTailCall) {
343 // Remove superfluous `br` terminators from the end of the Split blocks
344 // NOTE: Removing terminator removes the SplitBlock from the TailBB's
345 // predecessors. Therefore we must get complete list of Splits before
346 // attempting removal.
347 SmallVector Splits(predecessors((TailBB)));
348 assert(Splits.size() == 2 && "Expected exactly 2 splits!");
349 for (unsigned i = 0; i < Splits.size(); i++)
350 Splits[i]->getTerminator()->eraseFromParent();
351
352 // Erase the tail block once done with musttail patching
353 TailBB->eraseFromParent();
354 return;
355 }
292356 Instr->eraseFromParent();
293 NumCallSiteSplit++;
294357 }
295358
296359 // Return true if the call-site has an argument which is a PHI with only
368431 Function *Callee = CS.getCalledFunction();
369432 if (!Callee || Callee->isDeclaration())
370433 continue;
434
435 // Successful musttail call-site splits result in erased CI and erased BB.
436 // Check if such path is possible before attempting the splitting.
437 bool IsMustTail = CS.isMustTailCall();
438
371439 Changed |= tryToSplitCallSite(CS);
440
441 // There're no interesting instructions after this. The call site
442 // itself might have been erased on splitting.
443 if (IsMustTail)
444 break;
372445 }
373446 }
374447 return Changed;
0 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
1 ; RUN: opt < %s -callsite-splitting -S | FileCheck %s
2
3 define i8* @caller(i8* %a, i8* %b) {
4 ; CHECK-LABEL: @caller(
5 ; CHECK-NEXT: Top:
6 ; CHECK-NEXT: [[C:%.*]] = icmp eq i8* [[A:%.*]], null
7 ; CHECK-NEXT: br i1 [[C]], label [[TAIL_PREDBB1_SPLIT:%.*]], label [[TBB:%.*]]
8 ; CHECK: TBB:
9 ; CHECK-NEXT: [[C2:%.*]] = icmp eq i8* [[B:%.*]], null
10 ; CHECK-NEXT: br i1 [[C2]], label [[TAIL_PREDBB2_SPLIT:%.*]], label [[END:%.*]]
11 ; CHECK: Tail.predBB1.split:
12 ; CHECK-NEXT: [[TMP0:%.*]] = musttail call i8* @callee(i8* null, i8* [[B]])
13 ; CHECK-NEXT: [[CB1:%.*]] = bitcast i8* [[TMP0]] to i8*
14 ; CHECK-NEXT: ret i8* [[CB1]]
15 ; CHECK: Tail.predBB2.split:
16 ; CHECK-NEXT: [[TMP1:%.*]] = musttail call i8* @callee(i8* nonnull [[A]], i8* null)
17 ; CHECK-NEXT: [[CB2:%.*]] = bitcast i8* [[TMP1]] to i8*
18 ; CHECK-NEXT: ret i8* [[CB2]]
19 ; CHECK: End:
20 ; CHECK-NEXT: ret i8* null
21 ;
22 Top:
23 %c = icmp eq i8* %a, null
24 br i1 %c, label %Tail, label %TBB
25 TBB:
26 %c2 = icmp eq i8* %b, null
27 br i1 %c2, label %Tail, label %End
28 Tail:
29 %ca = musttail call i8* @callee(i8* %a, i8* %b)
30 %cb = bitcast i8* %ca to i8*
31 ret i8* %cb
32 End:
33 ret i8* null
34 }
35
36 define i8* @callee(i8* %a, i8* %b) noinline {
37 ; CHECK-LABEL: define i8* @callee(
38 ; CHECK-NEXT: ret i8* [[A:%.*]]
39 ;
40 ret i8* %a
41 }
42
43 define i8* @no_cast_caller(i8* %a, i8* %b) {
44 ; CHECK-LABEL: @no_cast_caller(
45 ; CHECK-NEXT: Top:
46 ; CHECK-NEXT: [[C:%.*]] = icmp eq i8* [[A:%.*]], null
47 ; CHECK-NEXT: br i1 [[C]], label [[TAIL_PREDBB1_SPLIT:%.*]], label [[TBB:%.*]]
48 ; CHECK: TBB:
49 ; CHECK-NEXT: [[C2:%.*]] = icmp eq i8* [[B:%.*]], null
50 ; CHECK-NEXT: br i1 [[C2]], label [[TAIL_PREDBB2_SPLIT:%.*]], label [[END:%.*]]
51 ; CHECK: Tail.predBB1.split:
52 ; CHECK-NEXT: [[TMP0:%.*]] = musttail call i8* @callee(i8* null, i8* [[B]])
53 ; CHECK-NEXT: ret i8* [[TMP0]]
54 ; CHECK: Tail.predBB2.split:
55 ; CHECK-NEXT: [[TMP1:%.*]] = musttail call i8* @callee(i8* nonnull [[A]], i8* null)
56 ; CHECK-NEXT: ret i8* [[TMP1]]
57 ; CHECK: End:
58 ; CHECK-NEXT: ret i8* null
59 ;
60 Top:
61 %c = icmp eq i8* %a, null
62 br i1 %c, label %Tail, label %TBB
63 TBB:
64 %c2 = icmp eq i8* %b, null
65 br i1 %c2, label %Tail, label %End
66 Tail:
67 %ca = musttail call i8* @callee(i8* %a, i8* %b)
68 ret i8* %ca
69 End:
70 ret i8* null
71 }
72
73 define void @void_caller(i8* %a, i8* %b) {
74 ; CHECK-LABEL: @void_caller(
75 ; CHECK-NEXT: Top:
76 ; CHECK-NEXT: [[C:%.*]] = icmp eq i8* [[A:%.*]], null
77 ; CHECK-NEXT: br i1 [[C]], label [[TAIL_PREDBB1_SPLIT:%.*]], label [[TBB:%.*]]
78 ; CHECK: TBB:
79 ; CHECK-NEXT: [[C2:%.*]] = icmp eq i8* [[B:%.*]], null
80 ; CHECK-NEXT: br i1 [[C2]], label [[TAIL_PREDBB2_SPLIT:%.*]], label [[END:%.*]]
81 ; CHECK: Tail.predBB1.split:
82 ; CHECK-NEXT: musttail call void @void_callee(i8* null, i8* [[B]])
83 ; CHECK-NEXT: ret void
84 ; CHECK: Tail.predBB2.split:
85 ; CHECK-NEXT: musttail call void @void_callee(i8* nonnull [[A]], i8* null)
86 ; CHECK-NEXT: ret void
87 ; CHECK: End:
88 ; CHECK-NEXT: ret void
89 ;
90 Top:
91 %c = icmp eq i8* %a, null
92 br i1 %c, label %Tail, label %TBB
93 TBB:
94 %c2 = icmp eq i8* %b, null
95 br i1 %c2, label %Tail, label %End
96 Tail:
97 musttail call void @void_callee(i8* %a, i8* %b)
98 ret void
99 End:
100 ret void
101 }
102
103 define void @void_callee(i8* %a, i8* %b) noinline {
104 ; CHECK-LABEL: define void @void_callee(
105 ; CHECK-NEXT: ret void
106 ;
107 ret void
108 }