llvm.org GIT mirror llvm / 164e0c1
code hoisting pass based on GVN This pass hoists duplicated computations in the program. The primary goal of gvn-hoist is to reduce the size of functions before inline heuristics to reduce the total cost of function inlining. Pass written by Sebastian Pop, Aditya Kumar, Xiaoyu Hu, and Brian Rzycki. Important algorithmic contributions by Daniel Berlin under the form of reviews. Differential Revision: http://reviews.llvm.org/D19338 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@274305 91177308-0d34-0410-b5e6-96231b3b80d8 Sebastian Pop 4 years ago
10 changed file(s) with 1418 addition(s) and 4 deletion(s). Raw diff Collapse all Expand all
118118 void initializeEdgeBundlesPass(PassRegistry&);
119119 void initializeEfficiencySanitizerPass(PassRegistry&);
120120 void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry &);
121 void initializeGVNHoistLegacyPassPass(PassRegistry &);
121122 void initializeExpandISelPseudosPass(PassRegistry&);
122123 void initializeExpandPostRAPass(PassRegistry&);
123124 void initializeExternalAAWrapperPassPass(PassRegistry&);
157157 (void) llvm::createConstantHoistingPass();
158158 (void) llvm::createCodeGenPreparePass();
159159 (void) llvm::createEarlyCSEPass();
160 (void) llvm::createGVNHoistPass();
160161 (void) llvm::createMergedLoadStoreMotionPass();
161162 (void) llvm::createGVNPass();
162163 (void) llvm::createMemCpyOptPass();
5757 AliasAnalysis *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
5858 MemoryDependenceResults &getMemDep() const { return *MD; }
5959
60 private:
61 friend class gvn::GVNLegacyPass;
62
6360 struct Expression;
64 friend struct DenseMapInfo;
6561
6662 /// This class holds the mapping between values and value numbers. It is used
6763 /// as an efficient mechanism to determine the expression-wise equivalence of
10298 uint32_t getNextUnusedValueNumber() { return nextValueNumber; }
10399 void verifyRemoved(const Value *) const;
104100 };
101
102 private:
103 friend class gvn::GVNLegacyPass;
104 friend struct DenseMapInfo;
105105
106106 MemoryDependenceResults *MD;
107107 DominatorTree *DT;
227227 /// loads are eliminated by the pass.
228228 FunctionPass *createGVNPass(bool NoLoads = false);
229229
230 /// \brief A simple and fast domtree-based GVN pass to hoist common expressions
231 /// from sibling branches.
232 struct GVNHoistPass : PassInfoMixin {
233 /// \brief Run the pass over the function.
234 PreservedAnalyses run(Function &F, AnalysisManager &AM);
235 };
236
230237 }
231238
232239 #endif
325325
326326 //===----------------------------------------------------------------------===//
327327 //
328 // GVNHoist - This pass performs a simple and fast GVN pass over the dominator
329 // tree to hoist common expressions from sibling branches.
330 //
331 FunctionPass *createGVNHoistPass();
332
333 //===----------------------------------------------------------------------===//
334 //
328335 // MergedLoadStoreMotion - This pass merges loads and stores in diamonds. Loads
329336 // are hoisted into the header, while stores sink into the footer.
330337 //
127127 FUNCTION_PASS("dce", DCEPass())
128128 FUNCTION_PASS("dse", DSEPass())
129129 FUNCTION_PASS("early-cse", EarlyCSEPass())
130 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
130131 FUNCTION_PASS("instcombine", InstCombinePass())
131132 FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass())
132133 FUNCTION_PASS("float2int", Float2IntPass())
198198 FPM.add(createCFGSimplificationPass());
199199 FPM.add(createSROAPass());
200200 FPM.add(createEarlyCSEPass());
201 FPM.add(createGVNHoistPass());
201202 FPM.add(createLowerExpectIntrinsicPass());
202203 }
203204
1111 Float2Int.cpp
1212 GuardWidening.cpp
1313 GVN.cpp
14 GVNHoist.cpp
1415 InductiveRangeCheckElimination.cpp
1516 IndVarSimplify.cpp
1617 JumpThreading.cpp
0 //===- GVNHoist.cpp - Hoist scalar and load expressions -------------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass hoists expressions from branches to a common dominator. It uses
10 // GVN (global value numbering) to discover expressions computing the same
11 // values. The primary goal is to reduce the code size, and in some
12 // cases reduce critical path (by exposing more ILP).
13 // Hoisting may affect the performance in some cases. To mitigate that, hoisting
14 // is disabled in the following cases.
15 // 1. Scalars across calls.
16 // 2. geps when corresponding load/store cannot be hoisted.
17 //===----------------------------------------------------------------------===//
18
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/Analysis/ValueTracking.h"
23 #include "llvm/Transforms/Scalar.h"
24 #include "llvm/Transforms/Scalar/GVN.h"
25 #include "llvm/Transforms/Utils/MemorySSA.h"
26 #include
27 #include
28 #include
29
30 using namespace llvm;
31
32 #define DEBUG_TYPE "gvn-hoist"
33
34 STATISTIC(NumHoisted, "Number of instructions hoisted");
35 STATISTIC(NumRemoved, "Number of instructions removed");
36 STATISTIC(NumLoadsHoisted, "Number of loads hoisted");
37 STATISTIC(NumLoadsRemoved, "Number of loads removed");
38 STATISTIC(NumStoresHoisted, "Number of stores hoisted");
39 STATISTIC(NumStoresRemoved, "Number of stores removed");
40 STATISTIC(NumCallsHoisted, "Number of calls hoisted");
41 STATISTIC(NumCallsRemoved, "Number of calls removed");
42
43 static cl::opt
44 MaxHoistedThreshold("gvn-max-hoisted", cl::Hidden, cl::init(-1),
45 cl::desc("Max number of instructions to hoist "
46 "(default unlimited = -1)"));
47 static cl::opt MaxNumberOfBBSInPath(
48 "gvn-hoist-max-bbs", cl::Hidden, cl::init(4),
49 cl::desc("Max number of basic blocks on the path between "
50 "hoisting locations (default = 4, unlimited = -1)"));
51
52 static int HoistedCtr = 0;
53
54 namespace {
55
56 // Provides a sorting function based on the execution order of two instructions.
57 struct SortByDFSIn {
58 private:
59 DenseMap &DFSNumber;
60
61 public:
62 SortByDFSIn(DenseMap &D) : DFSNumber(D) {}
63
64 // Returns true when A executes before B.
65 bool operator()(const Instruction *A, const Instruction *B) const {
66 assert(A != B);
67 const BasicBlock *BA = A->getParent();
68 const BasicBlock *BB = B->getParent();
69 unsigned NA = DFSNumber[BA];
70 unsigned NB = DFSNumber[BB];
71 if (NA < NB)
72 return true;
73 if (NA == NB) {
74 // Sort them in the order they occur in the same basic block.
75 BasicBlock::const_iterator AI(A), BI(B);
76 return std::distance(AI, BI) < 0;
77 }
78 return false;
79 }
80 };
81
82 // A map from a VN (value number) to all the instructions with that VN.
83 typedef DenseMap> VNtoInsns;
84
85 // Records all scalar instructions candidate for code hoisting.
86 class InsnInfo {
87 VNtoInsns VNtoScalars;
88
89 public:
90 // Inserts I and its value number in VNtoScalars.
91 void insert(Instruction *I, GVN::ValueTable &VN) {
92 // Scalar instruction.
93 unsigned V = VN.lookupOrAdd(I);
94 VNtoScalars[V].push_back(I);
95 }
96
97 const VNtoInsns &getVNTable() const { return VNtoScalars; }
98 };
99
100 // Records all load instructions candidate for code hoisting.
101 class LoadInfo {
102 VNtoInsns VNtoLoads;
103
104 public:
105 // Insert Load and the value number of its memory address in VNtoLoads.
106 void insert(LoadInst *Load, GVN::ValueTable &VN) {
107 if (Load->isSimple()) {
108 unsigned V = VN.lookupOrAdd(Load->getPointerOperand());
109 VNtoLoads[V].push_back(Load);
110 }
111 }
112
113 const VNtoInsns &getVNTable() const { return VNtoLoads; }
114 };
115
116 // Records all store instructions candidate for code hoisting.
117 class StoreInfo {
118 VNtoInsns VNtoStores;
119
120 public:
121 // Insert the Store and a hash number of the store address and the stored
122 // value in VNtoStores.
123 void insert(StoreInst *Store, GVN::ValueTable &VN) {
124 if (!Store->isSimple())
125 return;
126 // Hash the store address and the stored value.
127 Value *Ptr = Store->getPointerOperand();
128 Value *Val = Store->getValueOperand();
129 VNtoStores[hash_combine(VN.lookupOrAdd(Ptr), VN.lookupOrAdd(Val))]
130 .push_back(Store);
131 }
132
133 const VNtoInsns &getVNTable() const { return VNtoStores; }
134 };
135
136 // Records all call instructions candidate for code hoisting.
137 class CallInfo {
138 VNtoInsns VNtoCallsScalars;
139 VNtoInsns VNtoCallsLoads;
140 VNtoInsns VNtoCallsStores;
141
142 public:
143 // Insert Call and its value numbering in one of the VNtoCalls* containers.
144 void insert(CallInst *Call, GVN::ValueTable &VN) {
145 // A call that doesNotAccessMemory is handled as a Scalar,
146 // onlyReadsMemory will be handled as a Load instruction,
147 // all other calls will be handled as stores.
148 unsigned V = VN.lookupOrAdd(Call);
149
150 if (Call->doesNotAccessMemory())
151 VNtoCallsScalars[V].push_back(Call);
152 else if (Call->onlyReadsMemory())
153 VNtoCallsLoads[V].push_back(Call);
154 else
155 VNtoCallsStores[V].push_back(Call);
156 }
157
158 const VNtoInsns &getScalarVNTable() const { return VNtoCallsScalars; }
159
160 const VNtoInsns &getLoadVNTable() const { return VNtoCallsLoads; }
161
162 const VNtoInsns &getStoreVNTable() const { return VNtoCallsStores; }
163 };
164
165 typedef DenseMap BBSideEffectsSet;
166 typedef SmallVector SmallVecInsn;
167 typedef SmallVectorImpl SmallVecImplInsn;
168
169 // This pass hoists common computations across branches sharing common
170 // dominator. The primary goal is to reduce the code size, and in some
171 // cases reduce critical path (by exposing more ILP).
172 class GVNHoistLegacyPassImpl {
173 public:
174 GVN::ValueTable VN;
175 DominatorTree *DT;
176 AliasAnalysis *AA;
177 MemoryDependenceResults *MD;
178 DenseMap DFSNumber;
179 BBSideEffectsSet BBSideEffects;
180 MemorySSA *MSSA;
181 enum InsKind { Unknown, Scalar, Load, Store };
182
183 GVNHoistLegacyPassImpl(DominatorTree *Dt, AliasAnalysis *Aa,
184 MemoryDependenceResults *Md)
185 : DT(Dt), AA(Aa), MD(Md) {}
186
187 // Return true when there are exception handling in BB.
188 bool hasEH(const BasicBlock *BB) {
189 auto It = BBSideEffects.find(BB);
190 if (It != BBSideEffects.end())
191 return It->second;
192
193 if (BB->isEHPad() || BB->hasAddressTaken()) {
194 BBSideEffects[BB] = true;
195 return true;
196 }
197
198 if (BB->getTerminator()->mayThrow()) {
199 BBSideEffects[BB] = true;
200 return true;
201 }
202
203 BBSideEffects[BB] = false;
204 return false;
205 }
206
207 // Return true when all paths from A to the end of the function pass through
208 // either B or C.
209 bool hoistingFromAllPaths(const BasicBlock *A, const BasicBlock *B,
210 const BasicBlock *C) {
211 // We fully copy the WL in order to be able to remove items from it.
212 SmallPtrSet WL;
213 WL.insert(B);
214 WL.insert(C);
215
216 for (auto It = df_begin(A), E = df_end(A); It != E;) {
217 // There exists a path from A to the exit of the function if we are still
218 // iterating in DF traversal and we removed all instructions from the work
219 // list.
220 if (WL.empty())
221 return false;
222
223 const BasicBlock *BB = *It;
224 if (WL.erase(BB)) {
225 // Stop DFS traversal when BB is in the work list.
226 It.skipChildren();
227 continue;
228 }
229
230 // Check for end of function, calls that do not return, etc.
231 if (!isGuaranteedToTransferExecutionToSuccessor(BB->getTerminator()))
232 return false;
233
234 // Increment DFS traversal when not skipping children.
235 ++It;
236 }
237
238 return true;
239 }
240
241 // Each element of a hoisting list contains the basic block where to hoist and
242 // a list of instructions to be hoisted.
243 typedef std::pair HoistingPointInfo;
244 typedef SmallVector HoistingPointList;
245
246 // Return true when there are users of A in one of the BBs of Paths.
247 bool hasMemoryUse(MemoryAccess *A, const BasicBlock *PBB) {
248 Value::user_iterator UI = A->user_begin();
249 Value::user_iterator UE = A->user_end();
250 const BasicBlock *BBA = A->getBlock();
251 for (; UI != UE; ++UI)
252 if (MemoryAccess *UM = dyn_cast(*UI)) {
253 if (PBB == BBA)
254 if (MSSA->locallyDominates(UM, A))
255 return true;
256 if (PBB == UM->getBlock())
257 return true;
258 }
259 return false;
260 }
261
262 // Check whether it is possible to hoist in between NewHoistPt and BBInsn.
263 bool safeToHoist(const BasicBlock *NewHoistPt, const BasicBlock *BBInsn,
264 InsKind K, int &NBBsOnAllPaths, MemoryAccess *MemdefInsn,
265 BasicBlock *BBMemdefInsn, MemoryAccess *MemdefFirst,
266 BasicBlock *BBMemdefFirst) {
267 assert(DT->dominates(NewHoistPt, BBInsn) && "Invalid path");
268
269 // Record in Paths all basic blocks reachable in depth-first iteration on
270 // the inverse CFG from BBInsn to NewHoistPt. These blocks are all the
271 // blocks that may be executed between the execution of NewHoistPt and
272 // BBInsn. Hoisting an expression from BBInsn into NewHoistPt has to be safe
273 // on all execution paths.
274 for (auto I = idf_begin(BBInsn), E = idf_end(BBInsn); I != E;) {
275 if (*I == NewHoistPt) {
276 // Stop traversal when reaching NewHoistPt.
277 I.skipChildren();
278 continue;
279 }
280
281 // The safety checks for BBInsn will be handled separately.
282 if (*I != BBInsn) {
283 // Stop gathering blocks when it is not possible to hoist.
284 if (hasEH(*I))
285 return false;
286
287 // Check that we do not move a store past loads.
288 if (K == InsKind::Store) {
289 if (DT->dominates(BBMemdefInsn, NewHoistPt))
290 if (hasMemoryUse(MemdefInsn, *I))
291 return false;
292
293 if (DT->dominates(BBMemdefFirst, NewHoistPt))
294 if (hasMemoryUse(MemdefFirst, *I))
295 return false;
296 }
297 }
298 ++NBBsOnAllPaths;
299 ++I;
300 }
301
302 // Check whether there are too many blocks on the hoisting path.
303 if (MaxNumberOfBBSInPath != -1 && NBBsOnAllPaths >= MaxNumberOfBBSInPath)
304 return false;
305
306 return true;
307 }
308
309 // Return true when it is safe to hoist an instruction Insn to NewHoistPt and
310 // move the insertion point from HoistPt to NewHoistPt.
311 bool safeToHoist(const BasicBlock *NewHoistPt, const BasicBlock *HoistPt,
312 const Instruction *Insn, const Instruction *First, InsKind K,
313 int &NBBsOnAllPaths) {
314 if (hasEH(HoistPt))
315 return false;
316
317 const BasicBlock *BBInsn = Insn->getParent();
318 // When HoistPt already contains an instruction to be hoisted, the
319 // expression is needed on all paths.
320
321 // Check that the hoisted expression is needed on all paths: it is unsafe
322 // to hoist loads to a place where there may be a path not loading from
323 // the same address: for instance there may be a branch on which the
324 // address of the load may not be initialized. FIXME: at -Oz we may want
325 // to hoist scalars to a place where they are partially needed.
326 if (BBInsn != NewHoistPt &&
327 !hoistingFromAllPaths(NewHoistPt, HoistPt, BBInsn))
328 return false;
329
330 MemoryAccess *MemdefInsn = nullptr;
331 MemoryAccess *MemdefFirst = nullptr;
332 BasicBlock *BBMemdefInsn = nullptr;
333 BasicBlock *BBMemdefFirst = nullptr;
334
335 if (K != InsKind::Scalar) {
336 // For loads and stores, we check for dependences on the Memory SSA.
337 MemdefInsn = cast(MSSA->getMemoryAccess(Insn))
338 ->getDefiningAccess();
339 BBMemdefInsn = MemdefInsn->getBlock();
340
341 if (DT->properlyDominates(NewHoistPt, BBMemdefInsn))
342 // Cannot move Insn past BBMemdefInsn to NewHoistPt.
343 return false;
344
345 MemdefFirst = cast(MSSA->getMemoryAccess(First))
346 ->getDefiningAccess();
347 BBMemdefFirst = MemdefFirst->getBlock();
348
349 if (DT->properlyDominates(NewHoistPt, BBMemdefFirst))
350 // Cannot move First past BBMemdefFirst to NewHoistPt.
351 return false;
352 }
353
354 // Check for unsafe hoistings due to side effects.
355 if (!safeToHoist(NewHoistPt, HoistPt, K, NBBsOnAllPaths, MemdefInsn,
356 BBMemdefInsn, MemdefFirst, BBMemdefFirst) ||
357 !safeToHoist(NewHoistPt, BBInsn, K, NBBsOnAllPaths, MemdefInsn,
358 BBMemdefInsn, MemdefFirst, BBMemdefFirst))
359 return false;
360
361 // Safe to hoist scalars.
362 if (K == InsKind::Scalar)
363 return true;
364
365 if (DT->properlyDominates(BBMemdefInsn, NewHoistPt) &&
366 DT->properlyDominates(BBMemdefFirst, NewHoistPt))
367 return true;
368
369 const BasicBlock *BBFirst = First->getParent();
370 if (BBInsn == BBFirst)
371 return false;
372
373 assert(BBMemdefInsn == NewHoistPt || BBMemdefFirst == NewHoistPt);
374
375 if (BBInsn != NewHoistPt && BBFirst != NewHoistPt)
376 return true;
377
378 if (BBInsn == NewHoistPt) {
379 if (DT->properlyDominates(BBMemdefFirst, NewHoistPt))
380 return true;
381 assert(BBInsn == BBMemdefFirst);
382 if (MSSA->locallyDominates(MSSA->getMemoryAccess(Insn), MemdefFirst))
383 return false;
384 return true;
385 }
386
387 if (BBFirst == NewHoistPt) {
388 if (DT->properlyDominates(BBMemdefInsn, NewHoistPt))
389 return true;
390 assert(BBFirst == BBMemdefInsn);
391 if (MSSA->locallyDominates(MSSA->getMemoryAccess(First), MemdefInsn))
392 return false;
393 return true;
394 }
395
396 // No side effects: it is safe to hoist.
397 return true;
398 }
399
400 // Partition InstructionsToHoist into a set of candidates which can share a
401 // common hoisting point. The partitions are collected in HPL. IsScalar is
402 // true when the instructions in InstructionsToHoist are scalars. IsLoad is
403 // true when the InstructionsToHoist are loads, false when they are stores.
404 void partitionCandidates(SmallVecImplInsn &InstructionsToHoist,
405 HoistingPointList &HPL, InsKind K) {
406 // No need to sort for two instructions.
407 if (InstructionsToHoist.size() > 2) {
408 SortByDFSIn Pred(DFSNumber);
409 std::sort(InstructionsToHoist.begin(), InstructionsToHoist.end(), Pred);
410 }
411
412 // Create a work list of all the BB of the Insns to be hoisted.
413 SmallPtrSet WL;
414 SmallVecImplInsn::iterator II = InstructionsToHoist.begin();
415 SmallVecImplInsn::iterator Start = II;
416 BasicBlock *HoistPt = (*II)->getParent();
417 WL.insert((*II)->getParent());
418 int NBBsOnAllPaths = 0;
419
420 for (++II; II != InstructionsToHoist.end(); ++II) {
421 Instruction *Insn = *II;
422 BasicBlock *BB = Insn->getParent();
423 BasicBlock *NewHoistPt = DT->findNearestCommonDominator(HoistPt, BB);
424 WL.insert(BB);
425 if (safeToHoist(NewHoistPt, HoistPt, Insn, *Start, K, NBBsOnAllPaths)) {
426 // Extend HoistPt to NewHoistPt.
427 HoistPt = NewHoistPt;
428 continue;
429 }
430 // Not safe to hoist: save the previous work list and start over from BB.
431 if (std::distance(Start, II) > 1)
432 HPL.push_back(std::make_pair(HoistPt, SmallVecInsn(Start, II)));
433 else
434 WL.clear();
435
436 // We start over to compute HoistPt from BB.
437 Start = II;
438 HoistPt = BB;
439 NBBsOnAllPaths = 0;
440 }
441
442 // Save the last partition.
443 if (std::distance(Start, II) > 1)
444 HPL.push_back(std::make_pair(HoistPt, SmallVecInsn(Start, II)));
445 }
446
447 // Initialize HPL from Map.
448 void computeInsertionPoints(const VNtoInsns &Map, HoistingPointList &HPL,
449 InsKind K) {
450 for (VNtoInsns::const_iterator It = Map.begin(); It != Map.end(); ++It) {
451 if (MaxHoistedThreshold != -1 && ++HoistedCtr > MaxHoistedThreshold)
452 return;
453
454 const SmallVecInsn &V = It->second;
455 if (V.size() < 2)
456 continue;
457
458 // Compute the insertion point and the list of expressions to be hoisted.
459 SmallVecInsn InstructionsToHoist;
460 for (auto I : V)
461 if (!hasEH(I->getParent()))
462 InstructionsToHoist.push_back(I);
463
464 if (InstructionsToHoist.size())
465 partitionCandidates(InstructionsToHoist, HPL, K);
466 }
467 }
468
469 // Return true when all operands of Instr are available at insertion point
470 // HoistPt. When limiting the number of hoisted expressions, one could hoist
471 // a load without hoisting its access function. So before hoisting any
472 // expression, make sure that all its operands are available at insert point.
473 bool allOperandsAvailable(const Instruction *I,
474 const BasicBlock *HoistPt) const {
475 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
476 const Value *Op = I->getOperand(i);
477 const Instruction *Inst = dyn_cast(Op);
478 if (Inst && !DT->dominates(Inst->getParent(), HoistPt))
479 return false;
480 }
481
482 return true;
483 }
484
485 Instruction *firstOfTwo(Instruction *I, Instruction *J) const {
486 for (Instruction &I1 : *I->getParent())
487 if (&I1 == I || &I1 == J)
488 return &I1;
489 llvm_unreachable("Both I and J must be from same BB");
490 }
491
492 // Replace the use of From with To in Insn.
493 void replaceUseWith(Instruction *Insn, Value *From, Value *To) const {
494 for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
495 UI != UE;) {
496 Use &U = *UI++;
497 if (U.getUser() == Insn) {
498 U.set(To);
499 return;
500 }
501 }
502 llvm_unreachable("should replace exactly once");
503 }
504
505 bool makeOperandsAvailable(Instruction *Repl, BasicBlock *HoistPt) const {
506 // Check whether the GEP of a ld/st can be synthesized at HoistPt.
507 Instruction *Gep = nullptr;
508 Instruction *Val = nullptr;
509 if (LoadInst *Ld = dyn_cast(Repl))
510 Gep = dyn_cast(Ld->getPointerOperand());
511 if (StoreInst *St = dyn_cast(Repl)) {
512 Gep = dyn_cast(St->getPointerOperand());
513 Val = dyn_cast(St->getValueOperand());
514 }
515
516 if (!Gep || !isa(Gep))
517 return false;
518
519 // Check whether we can compute the Gep at HoistPt.
520 if (!allOperandsAvailable(Gep, HoistPt))
521 return false;
522
523 // Also check that the stored value is available.
524 if (Val && !allOperandsAvailable(Val, HoistPt))
525 return false;
526
527 // Copy the gep before moving the ld/st.
528 Instruction *ClonedGep = Gep->clone();
529 ClonedGep->insertBefore(HoistPt->getTerminator());
530 replaceUseWith(Repl, Gep, ClonedGep);
531
532 // Also copy Val when it is a gep: geps are not hoisted by default.
533 if (Val && isa(Val)) {
534 Instruction *ClonedVal = Val->clone();
535 ClonedVal->insertBefore(HoistPt->getTerminator());
536 replaceUseWith(Repl, Val, ClonedVal);
537 }
538
539 return true;
540 }
541
542 std::pair hoist(HoistingPointList &HPL) {
543 unsigned NI = 0, NL = 0, NS = 0, NC = 0, NR = 0;
544 for (const HoistingPointInfo &HP : HPL) {
545 // Find out whether we already have one of the instructions in HoistPt,
546 // in which case we do not have to move it.
547 BasicBlock *HoistPt = HP.first;
548 const SmallVecInsn &InstructionsToHoist = HP.second;
549 Instruction *Repl = nullptr;
550 for (Instruction *I : InstructionsToHoist)
551 if (I->getParent() == HoistPt) {
552 // If there are two instructions in HoistPt to be hoisted in place:
553 // update Repl to be the first one, such that we can rename the uses
554 // of the second based on the first.
555 Repl = !Repl ? I : firstOfTwo(Repl, I);
556 }
557
558 if (Repl) {
559 // Repl is already in HoistPt: it remains in place.
560 assert(allOperandsAvailable(Repl, HoistPt) &&
561 "instruction depends on operands that are not available");
562 } else {
563 // When we do not find Repl in HoistPt, select the first in the list
564 // and move it to HoistPt.
565 Repl = InstructionsToHoist.front();
566
567 // We can move Repl in HoistPt only when all operands are available.
568 // The order in which hoistings are done may influence the availability
569 // of operands.
570 if (!allOperandsAvailable(Repl, HoistPt) &&
571 !makeOperandsAvailable(Repl, HoistPt))
572 continue;
573 Repl->moveBefore(HoistPt->getTerminator());
574 }
575
576 if (isa(Repl))
577 ++NL;
578 else if (isa(Repl))
579 ++NS;
580 else if (isa(Repl))
581 ++NC;
582 else // Scalar
583 ++NI;
584
585 // Remove and rename all other instructions.
586 for (Instruction *I : InstructionsToHoist)
587 if (I != Repl) {
588 ++NR;
589 if (isa(Repl))
590 ++NumLoadsRemoved;
591 else if (isa(Repl))
592 ++NumStoresRemoved;
593 else if (isa(Repl))
594 ++NumCallsRemoved;
595 I->replaceAllUsesWith(Repl);
596 I->eraseFromParent();
597 }
598 }
599
600 NumHoisted += NL + NS + NC + NI;
601 NumRemoved += NR;
602 NumLoadsHoisted += NL;
603 NumStoresHoisted += NS;
604 NumCallsHoisted += NC;
605 return {NI, NL + NC + NS};
606 }
607
608 // Hoist all expressions. Returns Number of scalars hoisted
609 // and number of non-scalars hoisted.
610 std::pair hoistExpressions(Function &F) {
611 InsnInfo II;
612 LoadInfo LI;
613 StoreInfo SI;
614 CallInfo CI;
615 const bool OptForMinSize = F.optForMinSize();
616 for (BasicBlock *BB : depth_first(&F.getEntryBlock())) {
617 for (Instruction &I1 : *BB) {
618 if (LoadInst *Load = dyn_cast(&I1))
619 LI.insert(Load, VN);
620 else if (StoreInst *Store = dyn_cast(&I1))
621 SI.insert(Store, VN);
622 else if (CallInst *Call = dyn_cast(&I1)) {
623 if (IntrinsicInst *Intr = dyn_cast(Call)) {
624 if (isa(Intr) ||
625 Intr->getIntrinsicID() == Intrinsic::assume)
626 continue;
627 }
628 if (Call->mayHaveSideEffects()) {
629 if (!OptForMinSize)
630 break;
631 // We may continue hoisting across calls which write to memory.
632 if (Call->mayThrow())
633 break;
634 }
635 CI.insert(Call, VN);
636 } else if (OptForMinSize || !isa(&I1))
637 // Do not hoist scalars past calls that may write to memory because
638 // that could result in spills later. geps are handled separately.
639 // TODO: We can relax this for targets like AArch64 as they have more
640 // registers than X86.
641 II.insert(&I1, VN);
642 }
643 }
644
645 HoistingPointList HPL;
646 computeInsertionPoints(II.getVNTable(), HPL, InsKind::Scalar);
647 computeInsertionPoints(LI.getVNTable(), HPL, InsKind::Load);
648 computeInsertionPoints(SI.getVNTable(), HPL, InsKind::Store);
649 computeInsertionPoints(CI.getScalarVNTable(), HPL, InsKind::Scalar);
650 computeInsertionPoints(CI.getLoadVNTable(), HPL, InsKind::Load);
651 computeInsertionPoints(CI.getStoreVNTable(), HPL, InsKind::Store);
652 return hoist(HPL);
653 }
654
655 bool run(Function &F) {
656 VN.setDomTree(DT);
657 VN.setAliasAnalysis(AA);
658 VN.setMemDep(MD);
659 bool Res = false;
660
661 unsigned I = 0;
662 for (const BasicBlock *BB : depth_first(&F.getEntryBlock()))
663 DFSNumber.insert(std::make_pair(BB, ++I));
664
665 // FIXME: use lazy evaluation of VN to avoid the fix-point computation.
666 while (1) {
667 // FIXME: only compute MemorySSA once. We need to update the analysis in
668 // the same time as transforming the code.
669 MemorySSA M(F, AA, DT);
670 MSSA = &M;
671
672 auto HoistStat = hoistExpressions(F);
673 if (HoistStat.first + HoistStat.second == 0) {
674 return Res;
675 }
676 if (HoistStat.second > 0) {
677 // To address a limitation of the current GVN, we need to rerun the
678 // hoisting after we hoisted loads in order to be able to hoist all
679 // scalars dependent on the hoisted loads. Same for stores.
680 VN.clear();
681 }
682 Res = true;
683 }
684
685 return Res;
686 }
687 };
688
689 class GVNHoistLegacyPass : public FunctionPass {
690 public:
691 static char ID;
692
693 GVNHoistLegacyPass() : FunctionPass(ID) {
694 initializeGVNHoistLegacyPassPass(*PassRegistry::getPassRegistry());
695 }
696
697 bool runOnFunction(Function &F) override {
698 auto &DT = getAnalysis().getDomTree();
699 auto &AA = getAnalysis().getAAResults();
700 auto &MD = getAnalysis().getMemDep();
701
702 GVNHoistLegacyPassImpl G(&DT, &AA, &MD);
703 return G.run(F);
704 }
705
706 void getAnalysisUsage(AnalysisUsage &AU) const override {
707 AU.addRequired();
708 AU.addRequired();
709 AU.addRequired();
710 AU.addPreserved();
711 }
712 };
713 } // namespace
714
715 PreservedAnalyses GVNHoistPass::run(Function &F,
716 AnalysisManager &AM) {
717 DominatorTree &DT = AM.getResult(F);
718 AliasAnalysis &AA = AM.getResult(F);
719 MemoryDependenceResults &MD = AM.getResult(F);
720
721 GVNHoistLegacyPassImpl G(&DT, &AA, &MD);
722 if (!G.run(F))
723 return PreservedAnalyses::all();
724
725 PreservedAnalyses PA;
726 PA.preserve();
727 return PA;
728 }
729
730 char GVNHoistLegacyPass::ID = 0;
731 INITIALIZE_PASS_BEGIN(GVNHoistLegacyPass, "gvn-hoist",
732 "Early GVN Hoisting of Expressions", false, false)
733 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
734 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
735 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
736 INITIALIZE_PASS_END(GVNHoistLegacyPass, "gvn-hoist",
737 "Early GVN Hoisting of Expressions", false, false)
738
739 FunctionPass *llvm::createGVNHoistPass() { return new GVNHoistLegacyPass(); }
4343 initializeGuardWideningLegacyPassPass(Registry);
4444 initializeGVNLegacyPassPass(Registry);
4545 initializeEarlyCSELegacyPassPass(Registry);
46 initializeGVNHoistLegacyPassPass(Registry);
4647 initializeFlattenCFGPassPass(Registry);
4748 initializeInductiveRangeCheckEliminationPass(Registry);
4849 initializeIndVarSimplifyLegacyPassPass(Registry);
235236 unwrap(PM)->add(createEarlyCSEPass());
236237 }
237238
239 void LLVMAddGVNHoistLegacyPass(LLVMPassManagerRef PM) {
240 unwrap(PM)->add(createGVNHoistPass());
241 }
242
238243 void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM) {
239244 unwrap(PM)->add(createTypeBasedAAWrapperPass());
240245 }
0 ; RUN: opt -gvn-hoist -S < %s | FileCheck %s
1 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
2 target triple = "x86_64-unknown-linux-gnu"
3
4 @GlobalVar = internal global float 1.000000e+00
5
6 ; Check that all scalar expressions are hoisted.
7 ;
8 ; CHECK-LABEL: @scalarsHoisting
9 ; CHECK: fsub
10 ; CHECK: fmul
11 ; CHECK: fsub
12 ; CHECK: fmul
13 ; CHECK-NOT: fmul
14 ; CHECK-NOT: fsub
15 define float @scalarsHoisting(float %d, float %min, float %max, float %a) {
16 entry:
17 %div = fdiv float 1.000000e+00, %d
18 %cmp = fcmp oge float %div, 0.000000e+00
19 br i1 %cmp, label %if.then, label %if.else
20
21 if.then: ; preds = %entry
22 %sub = fsub float %min, %a
23 %mul = fmul float %sub, %div
24 %sub1 = fsub float %max, %a
25 %mul2 = fmul float %sub1, %div
26 br label %if.end
27
28 if.else: ; preds = %entry
29 %sub3 = fsub float %max, %a
30 %mul4 = fmul float %sub3, %div
31 %sub5 = fsub float %min, %a
32 %mul6 = fmul float %sub5, %div
33 br label %if.end
34
35 if.end: ; preds = %if.else, %if.then
36 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
37 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
38 %add = fadd float %tmax.0, %tmin.0
39 ret float %add
40 }
41
42 ; Check that all loads and scalars depending on the loads are hoisted.
43 ; Check that getelementptr computation gets hoisted before the load.
44 ;
45 ; CHECK-LABEL: @readsAndScalarsHoisting
46 ; CHECK: load
47 ; CHECK: load
48 ; CHECK: load
49 ; CHECK: fsub
50 ; CHECK: fmul
51 ; CHECK: fsub
52 ; CHECK: fmul
53 ; CHECK-NOT: load
54 ; CHECK-NOT: fmul
55 ; CHECK-NOT: fsub
56 define float @readsAndScalarsHoisting(float %d, float* %min, float* %max, float* %a) {
57 entry:
58 %div = fdiv float 1.000000e+00, %d
59 %cmp = fcmp oge float %div, 0.000000e+00
60 br i1 %cmp, label %if.then, label %if.else
61
62 if.then: ; preds = %entry
63 %A = getelementptr float, float* %min, i32 1
64 %0 = load float, float* %A, align 4
65 %1 = load float, float* %a, align 4
66 %sub = fsub float %0, %1
67 %mul = fmul float %sub, %div
68 %2 = load float, float* %max, align 4
69 %sub1 = fsub float %2, %1
70 %mul2 = fmul float %sub1, %div
71 br label %if.end
72
73 if.else: ; preds = %entry
74 %3 = load float, float* %max, align 4
75 %4 = load float, float* %a, align 4
76 %sub3 = fsub float %3, %4
77 %mul4 = fmul float %sub3, %div
78 %B = getelementptr float, float* %min, i32 1
79 %5 = load float, float* %B, align 4
80 %sub5 = fsub float %5, %4
81 %mul6 = fmul float %sub5, %div
82 br label %if.end
83
84 if.end: ; preds = %if.else, %if.then
85 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
86 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
87 %add = fadd float %tmax.0, %tmin.0
88 ret float %add
89 }
90
91 ; Check that we do not hoist loads after a store: the first two loads will be
92 ; hoisted, and then the third load will not be hoisted.
93 ;
94 ; CHECK-LABEL: @readsAndWrites
95 ; CHECK: load
96 ; CHECK: load
97 ; CHECK: fsub
98 ; CHECK: fmul
99 ; CHECK: store
100 ; CHECK: load
101 ; CHECK: fsub
102 ; CHECK: fmul
103 ; CHECK: load
104 ; CHECK: fsub
105 ; CHECK: fmul
106 ; CHECK-NOT: load
107 ; CHECK-NOT: fmul
108 ; CHECK-NOT: fsub
109 define float @readsAndWrites(float %d, float* %min, float* %max, float* %a) {
110 entry:
111 %div = fdiv float 1.000000e+00, %d
112 %cmp = fcmp oge float %div, 0.000000e+00
113 br i1 %cmp, label %if.then, label %if.else
114
115 if.then: ; preds = %entry
116 %0 = load float, float* %min, align 4
117 %1 = load float, float* %a, align 4
118 store float %0, float* @GlobalVar
119 %sub = fsub float %0, %1
120 %mul = fmul float %sub, %div
121 %2 = load float, float* %max, align 4
122 %sub1 = fsub float %2, %1
123 %mul2 = fmul float %sub1, %div
124 br label %if.end
125
126 if.else: ; preds = %entry
127 %3 = load float, float* %max, align 4
128 %4 = load float, float* %a, align 4
129 %sub3 = fsub float %3, %4
130 %mul4 = fmul float %sub3, %div
131 %5 = load float, float* %min, align 4
132 %sub5 = fsub float %5, %4
133 %mul6 = fmul float %sub5, %div
134 br label %if.end
135
136 if.end: ; preds = %if.else, %if.then
137 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
138 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
139 %add = fadd float %tmax.0, %tmin.0
140 ret float %add
141 }
142
143 ; Check that we do hoist loads when the store is above the insertion point.
144 ;
145 ; CHECK-LABEL: @readsAndWriteAboveInsertPt
146 ; CHECK: load
147 ; CHECK: load
148 ; CHECK: load
149 ; CHECK: fsub
150 ; CHECK: fsub
151 ; CHECK: fmul
152 ; CHECK: fmul
153 ; CHECK-NOT: load
154 ; CHECK-NOT: fmul
155 ; CHECK-NOT: fsub
156 define float @readsAndWriteAboveInsertPt(float %d, float* %min, float* %max, float* %a) {
157 entry:
158 %div = fdiv float 1.000000e+00, %d
159 store float 0.000000e+00, float* @GlobalVar
160 %cmp = fcmp oge float %div, 0.000000e+00
161 br i1 %cmp, label %if.then, label %if.else
162
163 if.then: ; preds = %entry
164 %0 = load float, float* %min, align 4
165 %1 = load float, float* %a, align 4
166 %sub = fsub float %0, %1
167 %mul = fmul float %sub, %div
168 %2 = load float, float* %max, align 4
169 %sub1 = fsub float %2, %1
170 %mul2 = fmul float %sub1, %div
171 br label %if.end
172
173 if.else: ; preds = %entry
174 %3 = load float, float* %max, align 4
175 %4 = load float, float* %a, align 4
176 %sub3 = fsub float %3, %4
177 %mul4 = fmul float %sub3, %div
178 %5 = load float, float* %min, align 4
179 %sub5 = fsub float %5, %4
180 %mul6 = fmul float %sub5, %div
181 br label %if.end
182
183 if.end: ; preds = %if.else, %if.then
184 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
185 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
186 %add = fadd float %tmax.0, %tmin.0
187 ret float %add
188 }
189
190 ; Check that dependent expressions are hoisted.
191 ; CHECK-LABEL: @dependentScalarsHoisting
192 ; CHECK: fsub
193 ; CHECK: fadd
194 ; CHECK: fdiv
195 ; CHECK: fmul
196 ; CHECK-NOT: fsub
197 ; CHECK-NOT: fadd
198 ; CHECK-NOT: fdiv
199 ; CHECK-NOT: fmul
200 define float @dependentScalarsHoisting(float %a, float %b, i1 %c) {
201 entry:
202 br i1 %c, label %if.then, label %if.else
203
204 if.then:
205 %d = fsub float %b, %a
206 %e = fadd float %d, %a
207 %f = fdiv float %e, %a
208 %g = fmul float %f, %a
209 br label %if.end
210
211 if.else:
212 %h = fsub float %b, %a
213 %i = fadd float %h, %a
214 %j = fdiv float %i, %a
215 %k = fmul float %j, %a
216 br label %if.end
217
218 if.end:
219 %r = phi float [ %g, %if.then ], [ %k, %if.else ]
220 ret float %r
221 }
222
223 ; Check that all independent expressions are hoisted.
224 ; CHECK-LABEL: @independentScalarsHoisting
225 ; CHECK: fmul
226 ; CHECK: fadd
227 ; CHECK: fdiv
228 ; CHECK: fsub
229 ; CHECK-NOT: fsub
230 ; CHECK-NOT: fdiv
231 ; CHECK-NOT: fmul
232 define float @independentScalarsHoisting(float %a, float %b, i1 %c) {
233 entry:
234 br i1 %c, label %if.then, label %if.else
235
236 if.then:
237 %d = fadd float %b, %a
238 %e = fsub float %b, %a
239 %f = fdiv float %b, %a
240 %g = fmul float %b, %a
241 br label %if.end
242
243 if.else:
244 %i = fadd float %b, %a
245 %h = fsub float %b, %a
246 %j = fdiv float %b, %a
247 %k = fmul float %b, %a
248 br label %if.end
249
250 if.end:
251 %p = phi float [ %d, %if.then ], [ %i, %if.else ]
252 %q = phi float [ %e, %if.then ], [ %h, %if.else ]
253 %r = phi float [ %f, %if.then ], [ %j, %if.else ]
254 %s = phi float [ %g, %if.then ], [ %k, %if.else ]
255 %t = fadd float %p, %q
256 %u = fadd float %r, %s
257 %v = fadd float %t, %u
258 ret float %v
259 }
260
261 ; Check that we hoist load and scalar expressions in triangles.
262 ; CHECK-LABEL: @triangleHoisting
263 ; CHECK: load
264 ; CHECK: load
265 ; CHECK: load
266 ; CHECK: fsub
267 ; CHECK: fsub
268 ; CHECK: fmul
269 ; CHECK: fmul
270 ; CHECK-NOT: load
271 ; CHECK-NOT: fmul
272 ; CHECK-NOT: fsub
273 define float @triangleHoisting(float %d, float* %min, float* %max, float* %a) {
274 entry:
275 %div = fdiv float 1.000000e+00, %d
276 %cmp = fcmp oge float %div, 0.000000e+00
277 br i1 %cmp, label %if.then, label %if.end
278
279 if.then: ; preds = %entry
280 %0 = load float, float* %min, align 4
281 %1 = load float, float* %a, align 4
282 %sub = fsub float %0, %1
283 %mul = fmul float %sub, %div
284 %2 = load float, float* %max, align 4
285 %sub1 = fsub float %2, %1
286 %mul2 = fmul float %sub1, %div
287 br label %if.end
288
289 if.end: ; preds = %entry
290 %p1 = phi float [ %mul2, %if.then ], [ 0.000000e+00, %entry ]
291 %p2 = phi float [ %mul, %if.then ], [ 0.000000e+00, %entry ]
292 %3 = load float, float* %max, align 4
293 %4 = load float, float* %a, align 4
294 %sub3 = fsub float %3, %4
295 %mul4 = fmul float %sub3, %div
296 %5 = load float, float* %min, align 4
297 %sub5 = fsub float %5, %4
298 %mul6 = fmul float %sub5, %div
299
300 %x = fadd float %p1, %mul6
301 %y = fadd float %p2, %mul4
302 %z = fadd float %x, %y
303 ret float %z
304 }
305
306 ; Check that we hoist load and scalar expressions in dominator.
307 ; CHECK-LABEL: @dominatorHoisting
308 ; CHECK: load
309 ; CHECK: load
310 ; CHECK: fsub
311 ; CHECK: fmul
312 ; CHECK: load
313 ; CHECK: fsub
314 ; CHECK: fmul
315 ; CHECK-NOT: load
316 ; CHECK-NOT: fmul
317 ; CHECK-NOT: fsub
318 define float @dominatorHoisting(float %d, float* %min, float* %max, float* %a) {
319 entry:
320 %div = fdiv float 1.000000e+00, %d
321 %0 = load float, float* %min, align 4
322 %1 = load float, float* %a, align 4
323 %sub = fsub float %0, %1
324 %mul = fmul float %sub, %div
325 %2 = load float, float* %max, align 4
326 %sub1 = fsub float %2, %1
327 %mul2 = fmul float %sub1, %div
328 %cmp = fcmp oge float %div, 0.000000e+00
329 br i1 %cmp, label %if.then, label %if.end
330
331 if.then: ; preds = %entry
332 %3 = load float, float* %max, align 4
333 %4 = load float, float* %a, align 4
334 %sub3 = fsub float %3, %4
335 %mul4 = fmul float %sub3, %div
336 %5 = load float, float* %min, align 4
337 %sub5 = fsub float %5, %4
338 %mul6 = fmul float %sub5, %div
339 br label %if.end
340
341 if.end: ; preds = %entry
342 %p1 = phi float [ %mul4, %if.then ], [ 0.000000e+00, %entry ]
343 %p2 = phi float [ %mul6, %if.then ], [ 0.000000e+00, %entry ]
344
345 %x = fadd float %p1, %mul2
346 %y = fadd float %p2, %mul
347 %z = fadd float %x, %y
348 ret float %z
349 }
350
351 ; Check that we hoist load and scalar expressions in dominator.
352 ; CHECK-LABEL: @domHoisting
353 ; CHECK: load
354 ; CHECK: load
355 ; CHECK: fsub
356 ; CHECK: fmul
357 ; CHECK: load
358 ; CHECK: fsub
359 ; CHECK: fmul
360 ; CHECK-NOT: load
361 ; CHECK-NOT: fmul
362 ; CHECK-NOT: fsub
363 define float @domHoisting(float %d, float* %min, float* %max, float* %a) {
364 entry:
365 %div = fdiv float 1.000000e+00, %d
366 %0 = load float, float* %min, align 4
367 %1 = load float, float* %a, align 4
368 %sub = fsub float %0, %1
369 %mul = fmul float %sub, %div
370 %2 = load float, float* %max, align 4
371 %sub1 = fsub float %2, %1
372 %mul2 = fmul float %sub1, %div
373 %cmp = fcmp oge float %div, 0.000000e+00
374 br i1 %cmp, label %if.then, label %if.else
375
376 if.then:
377 %3 = load float, float* %max, align 4
378 %4 = load float, float* %a, align 4
379 %sub3 = fsub float %3, %4
380 %mul4 = fmul float %sub3, %div
381 %5 = load float, float* %min, align 4
382 %sub5 = fsub float %5, %4
383 %mul6 = fmul float %sub5, %div
384 br label %if.end
385
386 if.else:
387 %6 = load float, float* %max, align 4
388 %7 = load float, float* %a, align 4
389 %sub9 = fsub float %6, %7
390 %mul10 = fmul float %sub9, %div
391 %8 = load float, float* %min, align 4
392 %sub12 = fsub float %8, %7
393 %mul13 = fmul float %sub12, %div
394 br label %if.end
395
396 if.end:
397 %p1 = phi float [ %mul4, %if.then ], [ %mul10, %if.else ]
398 %p2 = phi float [ %mul6, %if.then ], [ %mul13, %if.else ]
399
400 %x = fadd float %p1, %mul2
401 %y = fadd float %p2, %mul
402 %z = fadd float %x, %y
403 ret float %z
404 }
405
406 ; Check that we do not hoist loads past stores within a same basic block.
407 ; CHECK-LABEL: @noHoistInSingleBBWithStore
408 ; CHECK: load
409 ; CHECK: store
410 ; CHECK: load
411 ; CHECK: store
412 define i32 @noHoistInSingleBBWithStore() {
413 entry:
414 %D = alloca i32, align 4
415 %0 = bitcast i32* %D to i8*
416 %bf = load i8, i8* %0, align 4
417 %bf.clear = and i8 %bf, -3
418 store i8 %bf.clear, i8* %0, align 4
419 %bf1 = load i8, i8* %0, align 4
420 %bf.clear1 = and i8 %bf1, 1
421 store i8 %bf.clear1, i8* %0, align 4
422 ret i32 0
423 }
424
425 ; Check that we do not hoist loads past calls within a same basic block.
426 ; CHECK-LABEL: @noHoistInSingleBBWithCall
427 ; CHECK: load
428 ; CHECK: call
429 ; CHECK: load
430 declare void @foo()
431 define i32 @noHoistInSingleBBWithCall() {
432 entry:
433 %D = alloca i32, align 4
434 %0 = bitcast i32* %D to i8*
435 %bf = load i8, i8* %0, align 4
436 %bf.clear = and i8 %bf, -3
437 call void @foo()
438 %bf1 = load i8, i8* %0, align 4
439 %bf.clear1 = and i8 %bf1, 1
440 ret i32 0
441 }
442
443 ; Check that we do not hoist loads past stores in any branch of a diamond.
444 ; CHECK-LABEL: @noHoistInDiamondWithOneStore1
445 ; CHECK: fdiv
446 ; CHECK: fcmp
447 ; CHECK: br
448 define float @noHoistInDiamondWithOneStore1(float %d, float* %min, float* %max, float* %a) {
449 entry:
450 %div = fdiv float 1.000000e+00, %d
451 %cmp = fcmp oge float %div, 0.000000e+00
452 br i1 %cmp, label %if.then, label %if.else
453
454 if.then: ; preds = %entry
455 store float 0.000000e+00, float* @GlobalVar
456 %0 = load float, float* %min, align 4
457 %1 = load float, float* %a, align 4
458 %sub = fsub float %0, %1
459 %mul = fmul float %sub, %div
460 %2 = load float, float* %max, align 4
461 %sub1 = fsub float %2, %1
462 %mul2 = fmul float %sub1, %div
463 br label %if.end
464
465 if.else: ; preds = %entry
466 ; There are no side effects on the if.else branch.
467 %3 = load float, float* %max, align 4
468 %4 = load float, float* %a, align 4
469 %sub3 = fsub float %3, %4
470 %mul4 = fmul float %sub3, %div
471 %5 = load float, float* %min, align 4
472 %sub5 = fsub float %5, %4
473 %mul6 = fmul float %sub5, %div
474 br label %if.end
475
476 if.end: ; preds = %if.else, %if.then
477 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
478 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
479
480 %6 = load float, float* %max, align 4
481 %7 = load float, float* %a, align 4
482 %sub6 = fsub float %6, %7
483 %mul7 = fmul float %sub6, %div
484 %8 = load float, float* %min, align 4
485 %sub8 = fsub float %8, %7
486 %mul9 = fmul float %sub8, %div
487
488 %add = fadd float %tmax.0, %tmin.0
489 ret float %add
490 }
491
492 ; Check that we do not hoist loads past a store in any branch of a diamond.
493 ; CHECK-LABEL: @noHoistInDiamondWithOneStore2
494 ; CHECK: fdiv
495 ; CHECK: fcmp
496 ; CHECK: br
497 define float @noHoistInDiamondWithOneStore2(float %d, float* %min, float* %max, float* %a) {
498 entry:
499 %div = fdiv float 1.000000e+00, %d
500 %cmp = fcmp oge float %div, 0.000000e+00
501 br i1 %cmp, label %if.then, label %if.else
502
503 if.then: ; preds = %entry
504 ; There are no side effects on the if.then branch.
505 %0 = load float, float* %min, align 4
506 %1 = load float, float* %a, align 4
507 %sub = fsub float %0, %1
508 %mul = fmul float %sub, %div
509 %2 = load float, float* %max, align 4
510 %sub1 = fsub float %2, %1
511 %mul2 = fmul float %sub1, %div
512 br label %if.end
513
514 if.else: ; preds = %entry
515 store float 0.000000e+00, float* @GlobalVar
516 %3 = load float, float* %max, align 4
517 %4 = load float, float* %a, align 4
518 %sub3 = fsub float %3, %4
519 %mul4 = fmul float %sub3, %div
520 %5 = load float, float* %min, align 4
521 %sub5 = fsub float %5, %4
522 %mul6 = fmul float %sub5, %div
523 br label %if.end
524
525 if.end: ; preds = %if.else, %if.then
526 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
527 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
528
529 %6 = load float, float* %max, align 4
530 %7 = load float, float* %a, align 4
531 %sub6 = fsub float %6, %7
532 %mul7 = fmul float %sub6, %div
533 %8 = load float, float* %min, align 4
534 %sub8 = fsub float %8, %7
535 %mul9 = fmul float %sub8, %div
536
537 %add = fadd float %tmax.0, %tmin.0
538 ret float %add
539 }
540
541 ; Check that we do not hoist loads outside a loop containing stores.
542 ; CHECK-LABEL: @noHoistInLoopsWithStores
543 ; CHECK: fdiv
544 ; CHECK: fcmp
545 ; CHECK: br
546 define float @noHoistInLoopsWithStores(float %d, float* %min, float* %max, float* %a) {
547 entry:
548 %div = fdiv float 1.000000e+00, %d
549 %cmp = fcmp oge float %div, 0.000000e+00
550 br i1 %cmp, label %do.body, label %if.else
551
552 do.body:
553 %0 = load float, float* %min, align 4
554 %1 = load float, float* %a, align 4
555
556 ; It is unsafe to hoist the loads outside the loop because of the store.
557 store float 0.000000e+00, float* @GlobalVar
558
559 %sub = fsub float %0, %1
560 %mul = fmul float %sub, %div
561 %2 = load float, float* %max, align 4
562 %sub1 = fsub float %2, %1
563 %mul2 = fmul float %sub1, %div
564 br label %while.cond
565
566 while.cond:
567 %cmp1 = fcmp oge float %mul2, 0.000000e+00
568 br i1 %cmp1, label %if.end, label %do.body
569
570 if.else:
571 %3 = load float, float* %max, align 4
572 %4 = load float, float* %a, align 4
573 %sub3 = fsub float %3, %4
574 %mul4 = fmul float %sub3, %div
575 %5 = load float, float* %min, align 4
576 %sub5 = fsub float %5, %4
577 %mul6 = fmul float %sub5, %div
578 br label %if.end
579
580 if.end:
581 %tmax.0 = phi float [ %mul2, %while.cond ], [ %mul6, %if.else ]
582 %tmin.0 = phi float [ %mul, %while.cond ], [ %mul4, %if.else ]
583
584 %add = fadd float %tmax.0, %tmin.0
585 ret float %add
586 }
587
588 ; Check that we hoist stores: all the instructions from the then branch
589 ; should be hoisted.
590 ; CHECK-LABEL: @hoistStores
591 ; CHECK: zext
592 ; CHECK: trunc
593 ; CHECK: getelementptr
594 ; CHECK: load
595 ; CHECK: getelementptr
596 ; CHECK: store
597 ; CHECK: load
598 ; CHECK: load
599 ; CHECK: zext
600 ; CHECK: add
601 ; CHECK: store
602 ; CHECK: br
603 ; CHECK: if.then
604 ; CHECK: br
605
606 %struct.foo = type { i16* }
607
608 define void @hoistStores(%struct.foo* %s, i32* %coord, i1 zeroext %delta) {
609 entry:
610 %frombool = zext i1 %delta to i8
611 %tobool = trunc i8 %frombool to i1
612 br i1 %tobool, label %if.then, label %if.else
613
614 if.then: ; preds = %entry
615 %p = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
616 %0 = load i16*, i16** %p, align 8
617 %incdec.ptr = getelementptr inbounds i16, i16* %0, i32 1
618 store i16* %incdec.ptr, i16** %p, align 8
619 %1 = load i16, i16* %0, align 2
620 %conv = zext i16 %1 to i32
621 %2 = load i32, i32* %coord, align 4
622 %add = add i32 %2, %conv
623 store i32 %add, i32* %coord, align 4
624 br label %if.end
625
626 if.else: ; preds = %entry
627 %p1 = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
628 %3 = load i16*, i16** %p1, align 8
629 %incdec.ptr2 = getelementptr inbounds i16, i16* %3, i32 1
630 store i16* %incdec.ptr2, i16** %p1, align 8
631 %4 = load i16, i16* %3, align 2
632 %conv3 = zext i16 %4 to i32
633 %5 = load i32, i32* %coord, align 4
634 %add4 = add i32 %5, %conv3
635 store i32 %add4, i32* %coord, align 4
636 %6 = load i16*, i16** %p1, align 8
637 %incdec.ptr6 = getelementptr inbounds i16, i16* %6, i32 1
638 store i16* %incdec.ptr6, i16** %p1, align 8
639 %7 = load i16, i16* %6, align 2
640 %conv7 = zext i16 %7 to i32
641 %shl = shl i32 %conv7, 8
642 %8 = load i32, i32* %coord, align 4
643 %add8 = add i32 %8, %shl
644 store i32 %add8, i32* %coord, align 4
645 br label %if.end
646
647 if.end: ; preds = %if.else, %if.then
648 ret void
649 }