llvm.org GIT mirror llvm / 8e59cd1
[LICM] Hoist guards from non-header blocks This patch relaxes overconservative checks on whether or not we could write memory before we execute an instruction. This allows us to hoist guards out of loops even if they are not in the header block. Differential Revision: https://reviews.llvm.org/D50891 Reviewed By: fedor.sergeev git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@346643 91177308-0d34-0410-b5e6-96231b3b80d8 Max Kazantsev 11 months ago
6 changed file(s) with 229 addition(s) and 16 deletion(s). Raw diff Collapse all Expand all
113113 virtual bool isSpecialInstruction(const Instruction *Insn) const;
114114 };
115115
116 class MemoryWriteTracking : public InstructionPrecedenceTracking {
117 public:
118 MemoryWriteTracking(DominatorTree *DT) : InstructionPrecedenceTracking(DT) {}
119
120 /// Returns the topmost instruction that may write memory from the given
121 /// basic block. Returns nullptr if there is no such instructions in the block.
122 const Instruction *getFirstMemoryWrite(const BasicBlock *BB) {
123 return getFirstSpecialInstruction(BB);
124 }
125
126 /// Returns true if at least one instruction from the given basic block may
127 /// write memory.
128 bool mayWriteToMemory(const BasicBlock *BB) {
129 return hasSpecialInstructions(BB);
130 }
131
132 /// Returns true if the first memory writing instruction of Insn's block
133 /// exists and dominates Insn.
134 bool isDominatedByMemoryWriteFromSameBlock(const Instruction *Insn) {
135 return isPreceededBySpecialInstruction(Insn);
136 }
137
138 virtual bool isSpecialInstruction(const Instruction *Insn) const;
139 };
140
116141 } // llvm
117142
118143 #endif // LLVM_ANALYSIS_INSTRUCTIONPRECEDENCETRACKING_H
125125 // may throw.
126126 // Contains information about implicit control flow in this loop's blocks.
127127 mutable ImplicitControlFlowTracking ICF;
128 // Contains information about instruction that may possibly write memory.
129 mutable MemoryWriteTracking MW;
128130
129131 public:
130132 virtual bool blockMayThrow(const BasicBlock *BB) const;
137139 const DominatorTree *DT,
138140 const Loop *CurLoop) const;
139141
142 /// Returns true if we could not execute a memory-modifying instruction before
143 /// we enter \p BB under assumption that \p CurLoop is entered.
144 bool doesNotWriteMemoryBefore(const BasicBlock *BB, const Loop *CurLoop)
145 const;
146
147 /// Returns true if we could not execute a memory-modifying instruction before
148 /// we execute \p I under assumption that \p CurLoop is entered.
149 bool doesNotWriteMemoryBefore(const Instruction &I, const Loop *CurLoop)
150 const;
151
140152 /// Inform the safety info that we are planning to insert a new instruction
141153 /// into the basic block \p BB. It will make all cache updates to keep it
142154 /// correct after this insertion.
147159 /// this removal.
148160 void removeInstruction(const Instruction *Inst);
149161
150 ICFLoopSafetyInfo(DominatorTree *DT) : LoopSafetyInfo(), ICF(DT) {};
162 ICFLoopSafetyInfo(DominatorTree *DT) : LoopSafetyInfo(), ICF(DT), MW(DT) {};
151163
152164 virtual ~ICFLoopSafetyInfo() {};
153165 };
141141 }
142142 return true;
143143 }
144
145 bool MemoryWriteTracking::isSpecialInstruction(
146 const Instruction *Insn) const {
147 return Insn->mayWriteToMemory();
148 }
7171 void ICFLoopSafetyInfo::computeLoopSafetyInfo(const Loop *CurLoop) {
7272 assert(CurLoop != nullptr && "CurLoop can't be null");
7373 ICF.clear();
74 MW.clear();
7475 MayThrow = false;
7576 // Figure out the fact that at least one block may throw.
7677 for (auto &BB : CurLoop->blocks())
8384
8485 void ICFLoopSafetyInfo::insertInstructionTo(const BasicBlock *BB) {
8586 ICF.invalidateBlock(BB);
87 MW.invalidateBlock(BB);
8688 }
8789
8890 void ICFLoopSafetyInfo::removeInstruction(const Instruction *Inst) {
8991 // TODO: So far we just conservatively drop cache, but maybe we can not do it
9092 // when Inst is not an ICF instruction. Follow-up on that.
9193 ICF.invalidateBlock(Inst->getParent());
94 MW.invalidateBlock(Inst->getParent());
9295 }
9396
9497 void LoopSafetyInfo::computeBlockColors(const Loop *CurLoop) {
255258 allLoopPathsLeadToBlock(CurLoop, Inst.getParent(), DT);
256259 }
257260
261 bool ICFLoopSafetyInfo::doesNotWriteMemoryBefore(const BasicBlock *BB,
262 const Loop *CurLoop) const {
263 assert(CurLoop->contains(BB) && "Should only be called for loop blocks!");
264
265 // Fast path: there are no instructions before header.
266 if (BB == CurLoop->getHeader())
267 return true;
268
269 // Collect all transitive predecessors of BB in the same loop. This set will
270 // be a subset of the blocks within the loop.
271 SmallPtrSet Predecessors;
272 collectTransitivePredecessors(CurLoop, BB, Predecessors);
273 // Find if there any instruction in either predecessor that could write
274 // to memory.
275 for (auto *Pred : Predecessors)
276 if (MW.mayWriteToMemory(Pred))
277 return false;
278 return true;
279 }
280
281 bool ICFLoopSafetyInfo::doesNotWriteMemoryBefore(const Instruction &I,
282 const Loop *CurLoop) const {
283 auto *BB = I.getParent();
284 assert(CurLoop->contains(BB) && "Should only be called for loop blocks!");
285 return !MW.isDominatedByMemoryWriteFromSameBlock(&I) &&
286 doesNotWriteMemoryBefore(BB, CurLoop);
287 }
288
258289 namespace {
259290 struct MustExecutePrinter : public FunctionPass {
260291
462462 if (inSubLoop(BB, CurLoop, LI))
463463 continue;
464464
465 // Keep track of whether the prefix instructions could have written memory.
466 // TODO: This may be done smarter if we keep track of all throwing and
467 // mem-writing operations in every block, e.g. using something similar to
468 // isGuaranteedToExecute.
469 bool IsMemoryNotModified = CurLoop->getHeader() == BB;
470
471465 for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E;) {
472466 Instruction &I = *II++;
473467 // Try constant folding this instruction. If all the operands are
528522 if (((I.use_empty() &&
529523 match(&I, m_Intrinsic())) ||
530524 isGuard(&I)) &&
531 IsMemoryNotModified && CurLoop->hasLoopInvariantOperands(&I) &&
532 SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop)) {
525 CurLoop->hasLoopInvariantOperands(&I) &&
526 SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
527 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) {
533528 hoist(I, DT, CurLoop, SafetyInfo, ORE);
534529 Changed = true;
535530 continue;
536531 }
537
538 if (IsMemoryNotModified)
539 IsMemoryNotModified = !I.mayWriteToMemory();
540532 }
541533 }
542534
8484 }
8585
8686
87 ; TODO: We can also hoist this guard from mustexec non-header block.
8887 define void @test4(i1 %c, i32* %p) {
8988
9089 ; CHECK-LABEL: @test4(
9190 ; CHECK-LABEL: entry:
9291 ; CHECK: %a = load i32, i32* %p
9392 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
94 ; CHECK-LABEL: loop:
95 ; CHECK-LABEL: backedge:
96 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
93 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
94 ; CHECK-LABEL: loop:
95 ; CHECK-LABEL: backedge:
9796
9897 entry:
9998 br label %loop
189188 br label %backedge
190189
191190 backedge:
191 %loop_cond = icmp slt i32 %iv.next, 1000
192 br i1 %loop_cond, label %loop, label %exit
193
194 exit:
195 ret void
196 }
197
198 ; Check that we don't hoist across a store in the header.
199 define void @test4c(i1 %c, i32* %p, i8* noalias %s) {
200
201 ; CHECK-LABEL: @test4c(
202 ; CHECK-LABEL: entry:
203 ; CHECK: %a = load i32, i32* %p
204 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
205 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
206 ; CHECK-LABEL: loop:
207 ; CHECK-LABEL: backedge:
208
209 entry:
210 br label %loop
211
212 loop:
213 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
214 %iv.next = add i32 %iv, 1
215 store i8 0, i8* %s
216 br i1 %c, label %if.true, label %if.false
217
218 if.true:
219 br label %backedge
220
221 if.false:
222 br label %backedge
223
224 backedge:
225 %a = load i32, i32* %p
226 %invariant_cond = icmp ne i32 %a, 100
227 call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
228 %loop_cond = icmp slt i32 %iv.next, 1000
229 br i1 %loop_cond, label %loop, label %exit
230
231 exit:
232 ret void
233 }
234
235 ; Check that we don't hoist across a store in a conditionally execute block.
236 define void @test4d(i1 %c, i32* %p, i8* noalias %s) {
237
238 ; CHECK-LABEL: @test4d(
239 ; CHECK-LABEL: entry:
240 ; CHECK: %a = load i32, i32* %p
241 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
242 ; CHECK-LABEL: loop:
243 ; CHECK-LABEL: backedge:
244 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
245
246 entry:
247 br label %loop
248
249 loop:
250 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
251 %iv.next = add i32 %iv, 1
252 br i1 %c, label %if.true, label %if.false
253
254 if.true:
255 store i8 0, i8* %s
256 br label %backedge
257
258 if.false:
259 br label %backedge
260
261 backedge:
262 %a = load i32, i32* %p
263 %invariant_cond = icmp ne i32 %a, 100
264 call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
265 %loop_cond = icmp slt i32 %iv.next, 1000
266 br i1 %loop_cond, label %loop, label %exit
267
268 exit:
269 ret void
270 }
271
272 ; Check that we don't hoist across a store before the guard in the backedge.
273 define void @test4e(i1 %c, i32* %p, i8* noalias %s) {
274
275 ; CHECK-LABEL: @test4e(
276 ; CHECK-LABEL: entry:
277 ; CHECK: %a = load i32, i32* %p
278 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
279 ; CHECK: store i8 0, i8* %s
280 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
281 ; CHECK-LABEL: loop:
282 ; CHECK-LABEL: backedge:
283
284 entry:
285 br label %loop
286
287 loop:
288 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
289 %iv.next = add i32 %iv, 1
290 br i1 %c, label %if.true, label %if.false
291
292 if.true:
293 br label %backedge
294
295 if.false:
296 br label %backedge
297
298 backedge:
299 %a = load i32, i32* %p
300 %invariant_cond = icmp ne i32 %a, 100
301 store i8 0, i8* %s
302 call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
303 %loop_cond = icmp slt i32 %iv.next, 1000
304 br i1 %loop_cond, label %loop, label %exit
305
306 exit:
307 ret void
308 }
309
310 ; Check that we can hoist the guard in spite of store which happens after.
311 define void @test4f(i1 %c, i32* %p, i8* noalias %s) {
312
313 ; CHECK-LABEL: @test4f(
314 ; CHECK-LABEL: entry:
315 ; CHECK: %a = load i32, i32* %p
316 ; CHECK: %invariant_cond = icmp ne i32 %a, 100
317 ; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond)
318 ; CHECK-LABEL: loop:
319 ; CHECK-LABEL: backedge:
320
321 entry:
322 br label %loop
323
324 loop:
325 %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
326 %iv.next = add i32 %iv, 1
327 br i1 %c, label %if.true, label %if.false
328
329 if.true:
330 br label %backedge
331
332 if.false:
333 br label %backedge
334
335 backedge:
336 %a = load i32, i32* %p
337 %invariant_cond = icmp ne i32 %a, 100
338 call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
339 store i8 0, i8* %s
192340 %loop_cond = icmp slt i32 %iv.next, 1000
193341 br i1 %loop_cond, label %loop, label %exit
194342