llvm.org GIT mirror llvm / d88fd12
Revert "code hoisting pass based on GVN" This reverts commit r274305, since it breaks self-hosting: http://lab.llvm.org:8080/green/job/clang-stage1-configure-RA_build/22349/ http://lab.llvm.org:8011/builders/clang-x86_64-linux-selfhost-modules/builds/17232 Note that the blamelist on lab.llvm.org:8011 is incorrect. The previous build was r274299, but somehow r274305 wasn't included in the blamelist: http://lab.llvm.org:8011/builders/clang-x86_64-linux-selfhost-modules git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@274320 91177308-0d34-0410-b5e6-96231b3b80d8 Duncan P. N. Exon Smith 4 years ago
10 changed file(s) with 4 addition(s) and 1418 deletion(s). Raw diff Collapse all Expand all
118118 void initializeEdgeBundlesPass(PassRegistry&);
119119 void initializeEfficiencySanitizerPass(PassRegistry&);
120120 void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry &);
121 void initializeGVNHoistLegacyPassPass(PassRegistry &);
122121 void initializeExpandISelPseudosPass(PassRegistry&);
123122 void initializeExpandPostRAPass(PassRegistry&);
124123 void initializeExternalAAWrapperPassPass(PassRegistry&);
157157 (void) llvm::createConstantHoistingPass();
158158 (void) llvm::createCodeGenPreparePass();
159159 (void) llvm::createEarlyCSEPass();
160 (void) llvm::createGVNHoistPass();
161160 (void) llvm::createMergedLoadStoreMotionPass();
162161 (void) llvm::createGVNPass();
163162 (void) llvm::createMemCpyOptPass();
5757 AliasAnalysis *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
5858 MemoryDependenceResults &getMemDep() const { return *MD; }
5959
60 private:
61 friend class gvn::GVNLegacyPass;
62
6063 struct Expression;
64 friend struct DenseMapInfo;
6165
6266 /// This class holds the mapping between values and value numbers. It is used
6367 /// as an efficient mechanism to determine the expression-wise equivalence of
98102 uint32_t getNextUnusedValueNumber() { return nextValueNumber; }
99103 void verifyRemoved(const Value *) const;
100104 };
101
102 private:
103 friend class gvn::GVNLegacyPass;
104 friend struct DenseMapInfo;
105105
106106 MemoryDependenceResults *MD;
107107 DominatorTree *DT;
227227 /// loads are eliminated by the pass.
228228 FunctionPass *createGVNPass(bool NoLoads = false);
229229
230 /// \brief A simple and fast domtree-based GVN pass to hoist common expressions
231 /// from sibling branches.
232 struct GVNHoistPass : PassInfoMixin {
233 /// \brief Run the pass over the function.
234 PreservedAnalyses run(Function &F, AnalysisManager &AM);
235 };
236
237230 }
238231
239232 #endif
325325
326326 //===----------------------------------------------------------------------===//
327327 //
328 // GVNHoist - This pass performs a simple and fast GVN pass over the dominator
329 // tree to hoist common expressions from sibling branches.
330 //
331 FunctionPass *createGVNHoistPass();
332
333 //===----------------------------------------------------------------------===//
334 //
335328 // MergedLoadStoreMotion - This pass merges loads and stores in diamonds. Loads
336329 // are hoisted into the header, while stores sink into the footer.
337330 //
127127 FUNCTION_PASS("dce", DCEPass())
128128 FUNCTION_PASS("dse", DSEPass())
129129 FUNCTION_PASS("early-cse", EarlyCSEPass())
130 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
131130 FUNCTION_PASS("instcombine", InstCombinePass())
132131 FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass())
133132 FUNCTION_PASS("float2int", Float2IntPass())
198198 FPM.add(createCFGSimplificationPass());
199199 FPM.add(createSROAPass());
200200 FPM.add(createEarlyCSEPass());
201 FPM.add(createGVNHoistPass());
202201 FPM.add(createLowerExpectIntrinsicPass());
203202 }
204203
1111 Float2Int.cpp
1212 GuardWidening.cpp
1313 GVN.cpp
14 GVNHoist.cpp
1514 InductiveRangeCheckElimination.cpp
1615 IndVarSimplify.cpp
1716 JumpThreading.cpp
+0
-740
lib/Transforms/Scalar/GVNHoist.cpp less more
None //===- GVNHoist.cpp - Hoist scalar and load expressions -------------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass hoists expressions from branches to a common dominator. It uses
10 // GVN (global value numbering) to discover expressions computing the same
11 // values. The primary goal is to reduce the code size, and in some
12 // cases reduce critical path (by exposing more ILP).
13 // Hoisting may affect the performance in some cases. To mitigate that, hoisting
14 // is disabled in the following cases.
15 // 1. Scalars across calls.
16 // 2. geps when corresponding load/store cannot be hoisted.
17 //===----------------------------------------------------------------------===//
18
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/Analysis/ValueTracking.h"
23 #include "llvm/Transforms/Scalar.h"
24 #include "llvm/Transforms/Scalar/GVN.h"
25 #include "llvm/Transforms/Utils/MemorySSA.h"
26 #include
27 #include
28 #include
29
30 using namespace llvm;
31
32 #define DEBUG_TYPE "gvn-hoist"
33
34 STATISTIC(NumHoisted, "Number of instructions hoisted");
35 STATISTIC(NumRemoved, "Number of instructions removed");
36 STATISTIC(NumLoadsHoisted, "Number of loads hoisted");
37 STATISTIC(NumLoadsRemoved, "Number of loads removed");
38 STATISTIC(NumStoresHoisted, "Number of stores hoisted");
39 STATISTIC(NumStoresRemoved, "Number of stores removed");
40 STATISTIC(NumCallsHoisted, "Number of calls hoisted");
41 STATISTIC(NumCallsRemoved, "Number of calls removed");
42
43 static cl::opt
44 MaxHoistedThreshold("gvn-max-hoisted", cl::Hidden, cl::init(-1),
45 cl::desc("Max number of instructions to hoist "
46 "(default unlimited = -1)"));
47 static cl::opt MaxNumberOfBBSInPath(
48 "gvn-hoist-max-bbs", cl::Hidden, cl::init(4),
49 cl::desc("Max number of basic blocks on the path between "
50 "hoisting locations (default = 4, unlimited = -1)"));
51
52 static int HoistedCtr = 0;
53
54 namespace {
55
56 // Provides a sorting function based on the execution order of two instructions.
57 struct SortByDFSIn {
58 private:
59 DenseMap &DFSNumber;
60
61 public:
62 SortByDFSIn(DenseMap &D) : DFSNumber(D) {}
63
64 // Returns true when A executes before B.
65 bool operator()(const Instruction *A, const Instruction *B) const {
66 assert(A != B);
67 const BasicBlock *BA = A->getParent();
68 const BasicBlock *BB = B->getParent();
69 unsigned NA = DFSNumber[BA];
70 unsigned NB = DFSNumber[BB];
71 if (NA < NB)
72 return true;
73 if (NA == NB) {
74 // Sort them in the order they occur in the same basic block.
75 BasicBlock::const_iterator AI(A), BI(B);
76 return std::distance(AI, BI) < 0;
77 }
78 return false;
79 }
80 };
81
82 // A map from a VN (value number) to all the instructions with that VN.
83 typedef DenseMap> VNtoInsns;
84
85 // Records all scalar instructions candidate for code hoisting.
86 class InsnInfo {
87 VNtoInsns VNtoScalars;
88
89 public:
90 // Inserts I and its value number in VNtoScalars.
91 void insert(Instruction *I, GVN::ValueTable &VN) {
92 // Scalar instruction.
93 unsigned V = VN.lookupOrAdd(I);
94 VNtoScalars[V].push_back(I);
95 }
96
97 const VNtoInsns &getVNTable() const { return VNtoScalars; }
98 };
99
100 // Records all load instructions candidate for code hoisting.
101 class LoadInfo {
102 VNtoInsns VNtoLoads;
103
104 public:
105 // Insert Load and the value number of its memory address in VNtoLoads.
106 void insert(LoadInst *Load, GVN::ValueTable &VN) {
107 if (Load->isSimple()) {
108 unsigned V = VN.lookupOrAdd(Load->getPointerOperand());
109 VNtoLoads[V].push_back(Load);
110 }
111 }
112
113 const VNtoInsns &getVNTable() const { return VNtoLoads; }
114 };
115
116 // Records all store instructions candidate for code hoisting.
117 class StoreInfo {
118 VNtoInsns VNtoStores;
119
120 public:
121 // Insert the Store and a hash number of the store address and the stored
122 // value in VNtoStores.
123 void insert(StoreInst *Store, GVN::ValueTable &VN) {
124 if (!Store->isSimple())
125 return;
126 // Hash the store address and the stored value.
127 Value *Ptr = Store->getPointerOperand();
128 Value *Val = Store->getValueOperand();
129 VNtoStores[hash_combine(VN.lookupOrAdd(Ptr), VN.lookupOrAdd(Val))]
130 .push_back(Store);
131 }
132
133 const VNtoInsns &getVNTable() const { return VNtoStores; }
134 };
135
136 // Records all call instructions candidate for code hoisting.
137 class CallInfo {
138 VNtoInsns VNtoCallsScalars;
139 VNtoInsns VNtoCallsLoads;
140 VNtoInsns VNtoCallsStores;
141
142 public:
143 // Insert Call and its value numbering in one of the VNtoCalls* containers.
144 void insert(CallInst *Call, GVN::ValueTable &VN) {
145 // A call that doesNotAccessMemory is handled as a Scalar,
146 // onlyReadsMemory will be handled as a Load instruction,
147 // all other calls will be handled as stores.
148 unsigned V = VN.lookupOrAdd(Call);
149
150 if (Call->doesNotAccessMemory())
151 VNtoCallsScalars[V].push_back(Call);
152 else if (Call->onlyReadsMemory())
153 VNtoCallsLoads[V].push_back(Call);
154 else
155 VNtoCallsStores[V].push_back(Call);
156 }
157
158 const VNtoInsns &getScalarVNTable() const { return VNtoCallsScalars; }
159
160 const VNtoInsns &getLoadVNTable() const { return VNtoCallsLoads; }
161
162 const VNtoInsns &getStoreVNTable() const { return VNtoCallsStores; }
163 };
164
165 typedef DenseMap BBSideEffectsSet;
166 typedef SmallVector SmallVecInsn;
167 typedef SmallVectorImpl SmallVecImplInsn;
168
169 // This pass hoists common computations across branches sharing common
170 // dominator. The primary goal is to reduce the code size, and in some
171 // cases reduce critical path (by exposing more ILP).
172 class GVNHoistLegacyPassImpl {
173 public:
174 GVN::ValueTable VN;
175 DominatorTree *DT;
176 AliasAnalysis *AA;
177 MemoryDependenceResults *MD;
178 DenseMap DFSNumber;
179 BBSideEffectsSet BBSideEffects;
180 MemorySSA *MSSA;
181 enum InsKind { Unknown, Scalar, Load, Store };
182
183 GVNHoistLegacyPassImpl(DominatorTree *Dt, AliasAnalysis *Aa,
184 MemoryDependenceResults *Md)
185 : DT(Dt), AA(Aa), MD(Md) {}
186
187 // Return true when there are exception handling in BB.
188 bool hasEH(const BasicBlock *BB) {
189 auto It = BBSideEffects.find(BB);
190 if (It != BBSideEffects.end())
191 return It->second;
192
193 if (BB->isEHPad() || BB->hasAddressTaken()) {
194 BBSideEffects[BB] = true;
195 return true;
196 }
197
198 if (BB->getTerminator()->mayThrow()) {
199 BBSideEffects[BB] = true;
200 return true;
201 }
202
203 BBSideEffects[BB] = false;
204 return false;
205 }
206
207 // Return true when all paths from A to the end of the function pass through
208 // either B or C.
209 bool hoistingFromAllPaths(const BasicBlock *A, const BasicBlock *B,
210 const BasicBlock *C) {
211 // We fully copy the WL in order to be able to remove items from it.
212 SmallPtrSet WL;
213 WL.insert(B);
214 WL.insert(C);
215
216 for (auto It = df_begin(A), E = df_end(A); It != E;) {
217 // There exists a path from A to the exit of the function if we are still
218 // iterating in DF traversal and we removed all instructions from the work
219 // list.
220 if (WL.empty())
221 return false;
222
223 const BasicBlock *BB = *It;
224 if (WL.erase(BB)) {
225 // Stop DFS traversal when BB is in the work list.
226 It.skipChildren();
227 continue;
228 }
229
230 // Check for end of function, calls that do not return, etc.
231 if (!isGuaranteedToTransferExecutionToSuccessor(BB->getTerminator()))
232 return false;
233
234 // Increment DFS traversal when not skipping children.
235 ++It;
236 }
237
238 return true;
239 }
240
241 // Each element of a hoisting list contains the basic block where to hoist and
242 // a list of instructions to be hoisted.
243 typedef std::pair HoistingPointInfo;
244 typedef SmallVector HoistingPointList;
245
246 // Return true when there are users of A in one of the BBs of Paths.
247 bool hasMemoryUse(MemoryAccess *A, const BasicBlock *PBB) {
248 Value::user_iterator UI = A->user_begin();
249 Value::user_iterator UE = A->user_end();
250 const BasicBlock *BBA = A->getBlock();
251 for (; UI != UE; ++UI)
252 if (MemoryAccess *UM = dyn_cast(*UI)) {
253 if (PBB == BBA)
254 if (MSSA->locallyDominates(UM, A))
255 return true;
256 if (PBB == UM->getBlock())
257 return true;
258 }
259 return false;
260 }
261
262 // Check whether it is possible to hoist in between NewHoistPt and BBInsn.
263 bool safeToHoist(const BasicBlock *NewHoistPt, const BasicBlock *BBInsn,
264 InsKind K, int &NBBsOnAllPaths, MemoryAccess *MemdefInsn,
265 BasicBlock *BBMemdefInsn, MemoryAccess *MemdefFirst,
266 BasicBlock *BBMemdefFirst) {
267 assert(DT->dominates(NewHoistPt, BBInsn) && "Invalid path");
268
269 // Record in Paths all basic blocks reachable in depth-first iteration on
270 // the inverse CFG from BBInsn to NewHoistPt. These blocks are all the
271 // blocks that may be executed between the execution of NewHoistPt and
272 // BBInsn. Hoisting an expression from BBInsn into NewHoistPt has to be safe
273 // on all execution paths.
274 for (auto I = idf_begin(BBInsn), E = idf_end(BBInsn); I != E;) {
275 if (*I == NewHoistPt) {
276 // Stop traversal when reaching NewHoistPt.
277 I.skipChildren();
278 continue;
279 }
280
281 // The safety checks for BBInsn will be handled separately.
282 if (*I != BBInsn) {
283 // Stop gathering blocks when it is not possible to hoist.
284 if (hasEH(*I))
285 return false;
286
287 // Check that we do not move a store past loads.
288 if (K == InsKind::Store) {
289 if (DT->dominates(BBMemdefInsn, NewHoistPt))
290 if (hasMemoryUse(MemdefInsn, *I))
291 return false;
292
293 if (DT->dominates(BBMemdefFirst, NewHoistPt))
294 if (hasMemoryUse(MemdefFirst, *I))
295 return false;
296 }
297 }
298 ++NBBsOnAllPaths;
299 ++I;
300 }
301
302 // Check whether there are too many blocks on the hoisting path.
303 if (MaxNumberOfBBSInPath != -1 && NBBsOnAllPaths >= MaxNumberOfBBSInPath)
304 return false;
305
306 return true;
307 }
308
309 // Return true when it is safe to hoist an instruction Insn to NewHoistPt and
310 // move the insertion point from HoistPt to NewHoistPt.
311 bool safeToHoist(const BasicBlock *NewHoistPt, const BasicBlock *HoistPt,
312 const Instruction *Insn, const Instruction *First, InsKind K,
313 int &NBBsOnAllPaths) {
314 if (hasEH(HoistPt))
315 return false;
316
317 const BasicBlock *BBInsn = Insn->getParent();
318 // When HoistPt already contains an instruction to be hoisted, the
319 // expression is needed on all paths.
320
321 // Check that the hoisted expression is needed on all paths: it is unsafe
322 // to hoist loads to a place where there may be a path not loading from
323 // the same address: for instance there may be a branch on which the
324 // address of the load may not be initialized. FIXME: at -Oz we may want
325 // to hoist scalars to a place where they are partially needed.
326 if (BBInsn != NewHoistPt &&
327 !hoistingFromAllPaths(NewHoistPt, HoistPt, BBInsn))
328 return false;
329
330 MemoryAccess *MemdefInsn = nullptr;
331 MemoryAccess *MemdefFirst = nullptr;
332 BasicBlock *BBMemdefInsn = nullptr;
333 BasicBlock *BBMemdefFirst = nullptr;
334
335 if (K != InsKind::Scalar) {
336 // For loads and stores, we check for dependences on the Memory SSA.
337 MemdefInsn = cast(MSSA->getMemoryAccess(Insn))
338 ->getDefiningAccess();
339 BBMemdefInsn = MemdefInsn->getBlock();
340
341 if (DT->properlyDominates(NewHoistPt, BBMemdefInsn))
342 // Cannot move Insn past BBMemdefInsn to NewHoistPt.
343 return false;
344
345 MemdefFirst = cast(MSSA->getMemoryAccess(First))
346 ->getDefiningAccess();
347 BBMemdefFirst = MemdefFirst->getBlock();
348
349 if (DT->properlyDominates(NewHoistPt, BBMemdefFirst))
350 // Cannot move First past BBMemdefFirst to NewHoistPt.
351 return false;
352 }
353
354 // Check for unsafe hoistings due to side effects.
355 if (!safeToHoist(NewHoistPt, HoistPt, K, NBBsOnAllPaths, MemdefInsn,
356 BBMemdefInsn, MemdefFirst, BBMemdefFirst) ||
357 !safeToHoist(NewHoistPt, BBInsn, K, NBBsOnAllPaths, MemdefInsn,
358 BBMemdefInsn, MemdefFirst, BBMemdefFirst))
359 return false;
360
361 // Safe to hoist scalars.
362 if (K == InsKind::Scalar)
363 return true;
364
365 if (DT->properlyDominates(BBMemdefInsn, NewHoistPt) &&
366 DT->properlyDominates(BBMemdefFirst, NewHoistPt))
367 return true;
368
369 const BasicBlock *BBFirst = First->getParent();
370 if (BBInsn == BBFirst)
371 return false;
372
373 assert(BBMemdefInsn == NewHoistPt || BBMemdefFirst == NewHoistPt);
374
375 if (BBInsn != NewHoistPt && BBFirst != NewHoistPt)
376 return true;
377
378 if (BBInsn == NewHoistPt) {
379 if (DT->properlyDominates(BBMemdefFirst, NewHoistPt))
380 return true;
381 assert(BBInsn == BBMemdefFirst);
382 if (MSSA->locallyDominates(MSSA->getMemoryAccess(Insn), MemdefFirst))
383 return false;
384 return true;
385 }
386
387 if (BBFirst == NewHoistPt) {
388 if (DT->properlyDominates(BBMemdefInsn, NewHoistPt))
389 return true;
390 assert(BBFirst == BBMemdefInsn);
391 if (MSSA->locallyDominates(MSSA->getMemoryAccess(First), MemdefInsn))
392 return false;
393 return true;
394 }
395
396 // No side effects: it is safe to hoist.
397 return true;
398 }
399
400 // Partition InstructionsToHoist into a set of candidates which can share a
401 // common hoisting point. The partitions are collected in HPL. IsScalar is
402 // true when the instructions in InstructionsToHoist are scalars. IsLoad is
403 // true when the InstructionsToHoist are loads, false when they are stores.
404 void partitionCandidates(SmallVecImplInsn &InstructionsToHoist,
405 HoistingPointList &HPL, InsKind K) {
406 // No need to sort for two instructions.
407 if (InstructionsToHoist.size() > 2) {
408 SortByDFSIn Pred(DFSNumber);
409 std::sort(InstructionsToHoist.begin(), InstructionsToHoist.end(), Pred);
410 }
411
412 // Create a work list of all the BB of the Insns to be hoisted.
413 SmallPtrSet WL;
414 SmallVecImplInsn::iterator II = InstructionsToHoist.begin();
415 SmallVecImplInsn::iterator Start = II;
416 BasicBlock *HoistPt = (*II)->getParent();
417 WL.insert((*II)->getParent());
418 int NBBsOnAllPaths = 0;
419
420 for (++II; II != InstructionsToHoist.end(); ++II) {
421 Instruction *Insn = *II;
422 BasicBlock *BB = Insn->getParent();
423 BasicBlock *NewHoistPt = DT->findNearestCommonDominator(HoistPt, BB);
424 WL.insert(BB);
425 if (safeToHoist(NewHoistPt, HoistPt, Insn, *Start, K, NBBsOnAllPaths)) {
426 // Extend HoistPt to NewHoistPt.
427 HoistPt = NewHoistPt;
428 continue;
429 }
430 // Not safe to hoist: save the previous work list and start over from BB.
431 if (std::distance(Start, II) > 1)
432 HPL.push_back(std::make_pair(HoistPt, SmallVecInsn(Start, II)));
433 else
434 WL.clear();
435
436 // We start over to compute HoistPt from BB.
437 Start = II;
438 HoistPt = BB;
439 NBBsOnAllPaths = 0;
440 }
441
442 // Save the last partition.
443 if (std::distance(Start, II) > 1)
444 HPL.push_back(std::make_pair(HoistPt, SmallVecInsn(Start, II)));
445 }
446
447 // Initialize HPL from Map.
448 void computeInsertionPoints(const VNtoInsns &Map, HoistingPointList &HPL,
449 InsKind K) {
450 for (VNtoInsns::const_iterator It = Map.begin(); It != Map.end(); ++It) {
451 if (MaxHoistedThreshold != -1 && ++HoistedCtr > MaxHoistedThreshold)
452 return;
453
454 const SmallVecInsn &V = It->second;
455 if (V.size() < 2)
456 continue;
457
458 // Compute the insertion point and the list of expressions to be hoisted.
459 SmallVecInsn InstructionsToHoist;
460 for (auto I : V)
461 if (!hasEH(I->getParent()))
462 InstructionsToHoist.push_back(I);
463
464 if (InstructionsToHoist.size())
465 partitionCandidates(InstructionsToHoist, HPL, K);
466 }
467 }
468
469 // Return true when all operands of Instr are available at insertion point
470 // HoistPt. When limiting the number of hoisted expressions, one could hoist
471 // a load without hoisting its access function. So before hoisting any
472 // expression, make sure that all its operands are available at insert point.
473 bool allOperandsAvailable(const Instruction *I,
474 const BasicBlock *HoistPt) const {
475 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
476 const Value *Op = I->getOperand(i);
477 const Instruction *Inst = dyn_cast(Op);
478 if (Inst && !DT->dominates(Inst->getParent(), HoistPt))
479 return false;
480 }
481
482 return true;
483 }
484
485 Instruction *firstOfTwo(Instruction *I, Instruction *J) const {
486 for (Instruction &I1 : *I->getParent())
487 if (&I1 == I || &I1 == J)
488 return &I1;
489 llvm_unreachable("Both I and J must be from same BB");
490 }
491
492 // Replace the use of From with To in Insn.
493 void replaceUseWith(Instruction *Insn, Value *From, Value *To) const {
494 for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
495 UI != UE;) {
496 Use &U = *UI++;
497 if (U.getUser() == Insn) {
498 U.set(To);
499 return;
500 }
501 }
502 llvm_unreachable("should replace exactly once");
503 }
504
505 bool makeOperandsAvailable(Instruction *Repl, BasicBlock *HoistPt) const {
506 // Check whether the GEP of a ld/st can be synthesized at HoistPt.
507 Instruction *Gep = nullptr;
508 Instruction *Val = nullptr;
509 if (LoadInst *Ld = dyn_cast(Repl))
510 Gep = dyn_cast(Ld->getPointerOperand());
511 if (StoreInst *St = dyn_cast(Repl)) {
512 Gep = dyn_cast(St->getPointerOperand());
513 Val = dyn_cast(St->getValueOperand());
514 }
515
516 if (!Gep || !isa(Gep))
517 return false;
518
519 // Check whether we can compute the Gep at HoistPt.
520 if (!allOperandsAvailable(Gep, HoistPt))
521 return false;
522
523 // Also check that the stored value is available.
524 if (Val && !allOperandsAvailable(Val, HoistPt))
525 return false;
526
527 // Copy the gep before moving the ld/st.
528 Instruction *ClonedGep = Gep->clone();
529 ClonedGep->insertBefore(HoistPt->getTerminator());
530 replaceUseWith(Repl, Gep, ClonedGep);
531
532 // Also copy Val when it is a gep: geps are not hoisted by default.
533 if (Val && isa(Val)) {
534 Instruction *ClonedVal = Val->clone();
535 ClonedVal->insertBefore(HoistPt->getTerminator());
536 replaceUseWith(Repl, Val, ClonedVal);
537 }
538
539 return true;
540 }
541
542 std::pair hoist(HoistingPointList &HPL) {
543 unsigned NI = 0, NL = 0, NS = 0, NC = 0, NR = 0;
544 for (const HoistingPointInfo &HP : HPL) {
545 // Find out whether we already have one of the instructions in HoistPt,
546 // in which case we do not have to move it.
547 BasicBlock *HoistPt = HP.first;
548 const SmallVecInsn &InstructionsToHoist = HP.second;
549 Instruction *Repl = nullptr;
550 for (Instruction *I : InstructionsToHoist)
551 if (I->getParent() == HoistPt) {
552 // If there are two instructions in HoistPt to be hoisted in place:
553 // update Repl to be the first one, such that we can rename the uses
554 // of the second based on the first.
555 Repl = !Repl ? I : firstOfTwo(Repl, I);
556 }
557
558 if (Repl) {
559 // Repl is already in HoistPt: it remains in place.
560 assert(allOperandsAvailable(Repl, HoistPt) &&
561 "instruction depends on operands that are not available");
562 } else {
563 // When we do not find Repl in HoistPt, select the first in the list
564 // and move it to HoistPt.
565 Repl = InstructionsToHoist.front();
566
567 // We can move Repl in HoistPt only when all operands are available.
568 // The order in which hoistings are done may influence the availability
569 // of operands.
570 if (!allOperandsAvailable(Repl, HoistPt) &&
571 !makeOperandsAvailable(Repl, HoistPt))
572 continue;
573 Repl->moveBefore(HoistPt->getTerminator());
574 }
575
576 if (isa(Repl))
577 ++NL;
578 else if (isa(Repl))
579 ++NS;
580 else if (isa(Repl))
581 ++NC;
582 else // Scalar
583 ++NI;
584
585 // Remove and rename all other instructions.
586 for (Instruction *I : InstructionsToHoist)
587 if (I != Repl) {
588 ++NR;
589 if (isa(Repl))
590 ++NumLoadsRemoved;
591 else if (isa(Repl))
592 ++NumStoresRemoved;
593 else if (isa(Repl))
594 ++NumCallsRemoved;
595 I->replaceAllUsesWith(Repl);
596 I->eraseFromParent();
597 }
598 }
599
600 NumHoisted += NL + NS + NC + NI;
601 NumRemoved += NR;
602 NumLoadsHoisted += NL;
603 NumStoresHoisted += NS;
604 NumCallsHoisted += NC;
605 return {NI, NL + NC + NS};
606 }
607
608 // Hoist all expressions. Returns Number of scalars hoisted
609 // and number of non-scalars hoisted.
610 std::pair hoistExpressions(Function &F) {
611 InsnInfo II;
612 LoadInfo LI;
613 StoreInfo SI;
614 CallInfo CI;
615 const bool OptForMinSize = F.optForMinSize();
616 for (BasicBlock *BB : depth_first(&F.getEntryBlock())) {
617 for (Instruction &I1 : *BB) {
618 if (LoadInst *Load = dyn_cast(&I1))
619 LI.insert(Load, VN);
620 else if (StoreInst *Store = dyn_cast(&I1))
621 SI.insert(Store, VN);
622 else if (CallInst *Call = dyn_cast(&I1)) {
623 if (IntrinsicInst *Intr = dyn_cast(Call)) {
624 if (isa(Intr) ||
625 Intr->getIntrinsicID() == Intrinsic::assume)
626 continue;
627 }
628 if (Call->mayHaveSideEffects()) {
629 if (!OptForMinSize)
630 break;
631 // We may continue hoisting across calls which write to memory.
632 if (Call->mayThrow())
633 break;
634 }
635 CI.insert(Call, VN);
636 } else if (OptForMinSize || !isa(&I1))
637 // Do not hoist scalars past calls that may write to memory because
638 // that could result in spills later. geps are handled separately.
639 // TODO: We can relax this for targets like AArch64 as they have more
640 // registers than X86.
641 II.insert(&I1, VN);
642 }
643 }
644
645 HoistingPointList HPL;
646 computeInsertionPoints(II.getVNTable(), HPL, InsKind::Scalar);
647 computeInsertionPoints(LI.getVNTable(), HPL, InsKind::Load);
648 computeInsertionPoints(SI.getVNTable(), HPL, InsKind::Store);
649 computeInsertionPoints(CI.getScalarVNTable(), HPL, InsKind::Scalar);
650 computeInsertionPoints(CI.getLoadVNTable(), HPL, InsKind::Load);
651 computeInsertionPoints(CI.getStoreVNTable(), HPL, InsKind::Store);
652 return hoist(HPL);
653 }
654
655 bool run(Function &F) {
656 VN.setDomTree(DT);
657 VN.setAliasAnalysis(AA);
658 VN.setMemDep(MD);
659 bool Res = false;
660
661 unsigned I = 0;
662 for (const BasicBlock *BB : depth_first(&F.getEntryBlock()))
663 DFSNumber.insert(std::make_pair(BB, ++I));
664
665 // FIXME: use lazy evaluation of VN to avoid the fix-point computation.
666 while (1) {
667 // FIXME: only compute MemorySSA once. We need to update the analysis in
668 // the same time as transforming the code.
669 MemorySSA M(F, AA, DT);
670 MSSA = &M;
671
672 auto HoistStat = hoistExpressions(F);
673 if (HoistStat.first + HoistStat.second == 0) {
674 return Res;
675 }
676 if (HoistStat.second > 0) {
677 // To address a limitation of the current GVN, we need to rerun the
678 // hoisting after we hoisted loads in order to be able to hoist all
679 // scalars dependent on the hoisted loads. Same for stores.
680 VN.clear();
681 }
682 Res = true;
683 }
684
685 return Res;
686 }
687 };
688
689 class GVNHoistLegacyPass : public FunctionPass {
690 public:
691 static char ID;
692
693 GVNHoistLegacyPass() : FunctionPass(ID) {
694 initializeGVNHoistLegacyPassPass(*PassRegistry::getPassRegistry());
695 }
696
697 bool runOnFunction(Function &F) override {
698 auto &DT = getAnalysis().getDomTree();
699 auto &AA = getAnalysis().getAAResults();
700 auto &MD = getAnalysis().getMemDep();
701
702 GVNHoistLegacyPassImpl G(&DT, &AA, &MD);
703 return G.run(F);
704 }
705
706 void getAnalysisUsage(AnalysisUsage &AU) const override {
707 AU.addRequired();
708 AU.addRequired();
709 AU.addRequired();
710 AU.addPreserved();
711 }
712 };
713 } // namespace
714
715 PreservedAnalyses GVNHoistPass::run(Function &F,
716 AnalysisManager &AM) {
717 DominatorTree &DT = AM.getResult(F);
718 AliasAnalysis &AA = AM.getResult(F);
719 MemoryDependenceResults &MD = AM.getResult(F);
720
721 GVNHoistLegacyPassImpl G(&DT, &AA, &MD);
722 if (!G.run(F))
723 return PreservedAnalyses::all();
724
725 PreservedAnalyses PA;
726 PA.preserve();
727 return PA;
728 }
729
730 char GVNHoistLegacyPass::ID = 0;
731 INITIALIZE_PASS_BEGIN(GVNHoistLegacyPass, "gvn-hoist",
732 "Early GVN Hoisting of Expressions", false, false)
733 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
734 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
735 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
736 INITIALIZE_PASS_END(GVNHoistLegacyPass, "gvn-hoist",
737 "Early GVN Hoisting of Expressions", false, false)
738
739 FunctionPass *llvm::createGVNHoistPass() { return new GVNHoistLegacyPass(); }
4343 initializeGuardWideningLegacyPassPass(Registry);
4444 initializeGVNLegacyPassPass(Registry);
4545 initializeEarlyCSELegacyPassPass(Registry);
46 initializeGVNHoistLegacyPassPass(Registry);
4746 initializeFlattenCFGPassPass(Registry);
4847 initializeInductiveRangeCheckEliminationPass(Registry);
4948 initializeIndVarSimplifyLegacyPassPass(Registry);
236235 unwrap(PM)->add(createEarlyCSEPass());
237236 }
238237
239 void LLVMAddGVNHoistLegacyPass(LLVMPassManagerRef PM) {
240 unwrap(PM)->add(createGVNHoistPass());
241 }
242
243238 void LLVMAddTypeBasedAliasAnalysisPass(LLVMPassManagerRef PM) {
244239 unwrap(PM)->add(createTypeBasedAAWrapperPass());
245240 }
+0
-650
test/Transforms/GVN/hoist.ll less more
None ; RUN: opt -gvn-hoist -S < %s | FileCheck %s
1 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
2 target triple = "x86_64-unknown-linux-gnu"
3
4 @GlobalVar = internal global float 1.000000e+00
5
6 ; Check that all scalar expressions are hoisted.
7 ;
8 ; CHECK-LABEL: @scalarsHoisting
9 ; CHECK: fsub
10 ; CHECK: fmul
11 ; CHECK: fsub
12 ; CHECK: fmul
13 ; CHECK-NOT: fmul
14 ; CHECK-NOT: fsub
15 define float @scalarsHoisting(float %d, float %min, float %max, float %a) {
16 entry:
17 %div = fdiv float 1.000000e+00, %d
18 %cmp = fcmp oge float %div, 0.000000e+00
19 br i1 %cmp, label %if.then, label %if.else
20
21 if.then: ; preds = %entry
22 %sub = fsub float %min, %a
23 %mul = fmul float %sub, %div
24 %sub1 = fsub float %max, %a
25 %mul2 = fmul float %sub1, %div
26 br label %if.end
27
28 if.else: ; preds = %entry
29 %sub3 = fsub float %max, %a
30 %mul4 = fmul float %sub3, %div
31 %sub5 = fsub float %min, %a
32 %mul6 = fmul float %sub5, %div
33 br label %if.end
34
35 if.end: ; preds = %if.else, %if.then
36 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
37 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
38 %add = fadd float %tmax.0, %tmin.0
39 ret float %add
40 }
41
42 ; Check that all loads and scalars depending on the loads are hoisted.
43 ; Check that getelementptr computation gets hoisted before the load.
44 ;
45 ; CHECK-LABEL: @readsAndScalarsHoisting
46 ; CHECK: load
47 ; CHECK: load
48 ; CHECK: load
49 ; CHECK: fsub
50 ; CHECK: fmul
51 ; CHECK: fsub
52 ; CHECK: fmul
53 ; CHECK-NOT: load
54 ; CHECK-NOT: fmul
55 ; CHECK-NOT: fsub
56 define float @readsAndScalarsHoisting(float %d, float* %min, float* %max, float* %a) {
57 entry:
58 %div = fdiv float 1.000000e+00, %d
59 %cmp = fcmp oge float %div, 0.000000e+00
60 br i1 %cmp, label %if.then, label %if.else
61
62 if.then: ; preds = %entry
63 %A = getelementptr float, float* %min, i32 1
64 %0 = load float, float* %A, align 4
65 %1 = load float, float* %a, align 4
66 %sub = fsub float %0, %1
67 %mul = fmul float %sub, %div
68 %2 = load float, float* %max, align 4
69 %sub1 = fsub float %2, %1
70 %mul2 = fmul float %sub1, %div
71 br label %if.end
72
73 if.else: ; preds = %entry
74 %3 = load float, float* %max, align 4
75 %4 = load float, float* %a, align 4
76 %sub3 = fsub float %3, %4
77 %mul4 = fmul float %sub3, %div
78 %B = getelementptr float, float* %min, i32 1
79 %5 = load float, float* %B, align 4
80 %sub5 = fsub float %5, %4
81 %mul6 = fmul float %sub5, %div
82 br label %if.end
83
84 if.end: ; preds = %if.else, %if.then
85 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
86 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
87 %add = fadd float %tmax.0, %tmin.0
88 ret float %add
89 }
90
91 ; Check that we do not hoist loads after a store: the first two loads will be
92 ; hoisted, and then the third load will not be hoisted.
93 ;
94 ; CHECK-LABEL: @readsAndWrites
95 ; CHECK: load
96 ; CHECK: load
97 ; CHECK: fsub
98 ; CHECK: fmul
99 ; CHECK: store
100 ; CHECK: load
101 ; CHECK: fsub
102 ; CHECK: fmul
103 ; CHECK: load
104 ; CHECK: fsub
105 ; CHECK: fmul
106 ; CHECK-NOT: load
107 ; CHECK-NOT: fmul
108 ; CHECK-NOT: fsub
109 define float @readsAndWrites(float %d, float* %min, float* %max, float* %a) {
110 entry:
111 %div = fdiv float 1.000000e+00, %d
112 %cmp = fcmp oge float %div, 0.000000e+00
113 br i1 %cmp, label %if.then, label %if.else
114
115 if.then: ; preds = %entry
116 %0 = load float, float* %min, align 4
117 %1 = load float, float* %a, align 4
118 store float %0, float* @GlobalVar
119 %sub = fsub float %0, %1
120 %mul = fmul float %sub, %div
121 %2 = load float, float* %max, align 4
122 %sub1 = fsub float %2, %1
123 %mul2 = fmul float %sub1, %div
124 br label %if.end
125
126 if.else: ; preds = %entry
127 %3 = load float, float* %max, align 4
128 %4 = load float, float* %a, align 4
129 %sub3 = fsub float %3, %4
130 %mul4 = fmul float %sub3, %div
131 %5 = load float, float* %min, align 4
132 %sub5 = fsub float %5, %4
133 %mul6 = fmul float %sub5, %div
134 br label %if.end
135
136 if.end: ; preds = %if.else, %if.then
137 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
138 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
139 %add = fadd float %tmax.0, %tmin.0
140 ret float %add
141 }
142
143 ; Check that we do hoist loads when the store is above the insertion point.
144 ;
145 ; CHECK-LABEL: @readsAndWriteAboveInsertPt
146 ; CHECK: load
147 ; CHECK: load
148 ; CHECK: load
149 ; CHECK: fsub
150 ; CHECK: fsub
151 ; CHECK: fmul
152 ; CHECK: fmul
153 ; CHECK-NOT: load
154 ; CHECK-NOT: fmul
155 ; CHECK-NOT: fsub
156 define float @readsAndWriteAboveInsertPt(float %d, float* %min, float* %max, float* %a) {
157 entry:
158 %div = fdiv float 1.000000e+00, %d
159 store float 0.000000e+00, float* @GlobalVar
160 %cmp = fcmp oge float %div, 0.000000e+00
161 br i1 %cmp, label %if.then, label %if.else
162
163 if.then: ; preds = %entry
164 %0 = load float, float* %min, align 4
165 %1 = load float, float* %a, align 4
166 %sub = fsub float %0, %1
167 %mul = fmul float %sub, %div
168 %2 = load float, float* %max, align 4
169 %sub1 = fsub float %2, %1
170 %mul2 = fmul float %sub1, %div
171 br label %if.end
172
173 if.else: ; preds = %entry
174 %3 = load float, float* %max, align 4
175 %4 = load float, float* %a, align 4
176 %sub3 = fsub float %3, %4
177 %mul4 = fmul float %sub3, %div
178 %5 = load float, float* %min, align 4
179 %sub5 = fsub float %5, %4
180 %mul6 = fmul float %sub5, %div
181 br label %if.end
182
183 if.end: ; preds = %if.else, %if.then
184 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
185 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
186 %add = fadd float %tmax.0, %tmin.0
187 ret float %add
188 }
189
190 ; Check that dependent expressions are hoisted.
191 ; CHECK-LABEL: @dependentScalarsHoisting
192 ; CHECK: fsub
193 ; CHECK: fadd
194 ; CHECK: fdiv
195 ; CHECK: fmul
196 ; CHECK-NOT: fsub
197 ; CHECK-NOT: fadd
198 ; CHECK-NOT: fdiv
199 ; CHECK-NOT: fmul
200 define float @dependentScalarsHoisting(float %a, float %b, i1 %c) {
201 entry:
202 br i1 %c, label %if.then, label %if.else
203
204 if.then:
205 %d = fsub float %b, %a
206 %e = fadd float %d, %a
207 %f = fdiv float %e, %a
208 %g = fmul float %f, %a
209 br label %if.end
210
211 if.else:
212 %h = fsub float %b, %a
213 %i = fadd float %h, %a
214 %j = fdiv float %i, %a
215 %k = fmul float %j, %a
216 br label %if.end
217
218 if.end:
219 %r = phi float [ %g, %if.then ], [ %k, %if.else ]
220 ret float %r
221 }
222
223 ; Check that all independent expressions are hoisted.
224 ; CHECK-LABEL: @independentScalarsHoisting
225 ; CHECK: fmul
226 ; CHECK: fadd
227 ; CHECK: fdiv
228 ; CHECK: fsub
229 ; CHECK-NOT: fsub
230 ; CHECK-NOT: fdiv
231 ; CHECK-NOT: fmul
232 define float @independentScalarsHoisting(float %a, float %b, i1 %c) {
233 entry:
234 br i1 %c, label %if.then, label %if.else
235
236 if.then:
237 %d = fadd float %b, %a
238 %e = fsub float %b, %a
239 %f = fdiv float %b, %a
240 %g = fmul float %b, %a
241 br label %if.end
242
243 if.else:
244 %i = fadd float %b, %a
245 %h = fsub float %b, %a
246 %j = fdiv float %b, %a
247 %k = fmul float %b, %a
248 br label %if.end
249
250 if.end:
251 %p = phi float [ %d, %if.then ], [ %i, %if.else ]
252 %q = phi float [ %e, %if.then ], [ %h, %if.else ]
253 %r = phi float [ %f, %if.then ], [ %j, %if.else ]
254 %s = phi float [ %g, %if.then ], [ %k, %if.else ]
255 %t = fadd float %p, %q
256 %u = fadd float %r, %s
257 %v = fadd float %t, %u
258 ret float %v
259 }
260
261 ; Check that we hoist load and scalar expressions in triangles.
262 ; CHECK-LABEL: @triangleHoisting
263 ; CHECK: load
264 ; CHECK: load
265 ; CHECK: load
266 ; CHECK: fsub
267 ; CHECK: fsub
268 ; CHECK: fmul
269 ; CHECK: fmul
270 ; CHECK-NOT: load
271 ; CHECK-NOT: fmul
272 ; CHECK-NOT: fsub
273 define float @triangleHoisting(float %d, float* %min, float* %max, float* %a) {
274 entry:
275 %div = fdiv float 1.000000e+00, %d
276 %cmp = fcmp oge float %div, 0.000000e+00
277 br i1 %cmp, label %if.then, label %if.end
278
279 if.then: ; preds = %entry
280 %0 = load float, float* %min, align 4
281 %1 = load float, float* %a, align 4
282 %sub = fsub float %0, %1
283 %mul = fmul float %sub, %div
284 %2 = load float, float* %max, align 4
285 %sub1 = fsub float %2, %1
286 %mul2 = fmul float %sub1, %div
287 br label %if.end
288
289 if.end: ; preds = %entry
290 %p1 = phi float [ %mul2, %if.then ], [ 0.000000e+00, %entry ]
291 %p2 = phi float [ %mul, %if.then ], [ 0.000000e+00, %entry ]
292 %3 = load float, float* %max, align 4
293 %4 = load float, float* %a, align 4
294 %sub3 = fsub float %3, %4
295 %mul4 = fmul float %sub3, %div
296 %5 = load float, float* %min, align 4
297 %sub5 = fsub float %5, %4
298 %mul6 = fmul float %sub5, %div
299
300 %x = fadd float %p1, %mul6
301 %y = fadd float %p2, %mul4
302 %z = fadd float %x, %y
303 ret float %z
304 }
305
306 ; Check that we hoist load and scalar expressions in dominator.
307 ; CHECK-LABEL: @dominatorHoisting
308 ; CHECK: load
309 ; CHECK: load
310 ; CHECK: fsub
311 ; CHECK: fmul
312 ; CHECK: load
313 ; CHECK: fsub
314 ; CHECK: fmul
315 ; CHECK-NOT: load
316 ; CHECK-NOT: fmul
317 ; CHECK-NOT: fsub
318 define float @dominatorHoisting(float %d, float* %min, float* %max, float* %a) {
319 entry:
320 %div = fdiv float 1.000000e+00, %d
321 %0 = load float, float* %min, align 4
322 %1 = load float, float* %a, align 4
323 %sub = fsub float %0, %1
324 %mul = fmul float %sub, %div
325 %2 = load float, float* %max, align 4
326 %sub1 = fsub float %2, %1
327 %mul2 = fmul float %sub1, %div
328 %cmp = fcmp oge float %div, 0.000000e+00
329 br i1 %cmp, label %if.then, label %if.end
330
331 if.then: ; preds = %entry
332 %3 = load float, float* %max, align 4
333 %4 = load float, float* %a, align 4
334 %sub3 = fsub float %3, %4
335 %mul4 = fmul float %sub3, %div
336 %5 = load float, float* %min, align 4
337 %sub5 = fsub float %5, %4
338 %mul6 = fmul float %sub5, %div
339 br label %if.end
340
341 if.end: ; preds = %entry
342 %p1 = phi float [ %mul4, %if.then ], [ 0.000000e+00, %entry ]
343 %p2 = phi float [ %mul6, %if.then ], [ 0.000000e+00, %entry ]
344
345 %x = fadd float %p1, %mul2
346 %y = fadd float %p2, %mul
347 %z = fadd float %x, %y
348 ret float %z
349 }
350
351 ; Check that we hoist load and scalar expressions in dominator.
352 ; CHECK-LABEL: @domHoisting
353 ; CHECK: load
354 ; CHECK: load
355 ; CHECK: fsub
356 ; CHECK: fmul
357 ; CHECK: load
358 ; CHECK: fsub
359 ; CHECK: fmul
360 ; CHECK-NOT: load
361 ; CHECK-NOT: fmul
362 ; CHECK-NOT: fsub
363 define float @domHoisting(float %d, float* %min, float* %max, float* %a) {
364 entry:
365 %div = fdiv float 1.000000e+00, %d
366 %0 = load float, float* %min, align 4
367 %1 = load float, float* %a, align 4
368 %sub = fsub float %0, %1
369 %mul = fmul float %sub, %div
370 %2 = load float, float* %max, align 4
371 %sub1 = fsub float %2, %1
372 %mul2 = fmul float %sub1, %div
373 %cmp = fcmp oge float %div, 0.000000e+00
374 br i1 %cmp, label %if.then, label %if.else
375
376 if.then:
377 %3 = load float, float* %max, align 4
378 %4 = load float, float* %a, align 4
379 %sub3 = fsub float %3, %4
380 %mul4 = fmul float %sub3, %div
381 %5 = load float, float* %min, align 4
382 %sub5 = fsub float %5, %4
383 %mul6 = fmul float %sub5, %div
384 br label %if.end
385
386 if.else:
387 %6 = load float, float* %max, align 4
388 %7 = load float, float* %a, align 4
389 %sub9 = fsub float %6, %7
390 %mul10 = fmul float %sub9, %div
391 %8 = load float, float* %min, align 4
392 %sub12 = fsub float %8, %7
393 %mul13 = fmul float %sub12, %div
394 br label %if.end
395
396 if.end:
397 %p1 = phi float [ %mul4, %if.then ], [ %mul10, %if.else ]
398 %p2 = phi float [ %mul6, %if.then ], [ %mul13, %if.else ]
399
400 %x = fadd float %p1, %mul2
401 %y = fadd float %p2, %mul
402 %z = fadd float %x, %y
403 ret float %z
404 }
405
406 ; Check that we do not hoist loads past stores within a same basic block.
407 ; CHECK-LABEL: @noHoistInSingleBBWithStore
408 ; CHECK: load
409 ; CHECK: store
410 ; CHECK: load
411 ; CHECK: store
412 define i32 @noHoistInSingleBBWithStore() {
413 entry:
414 %D = alloca i32, align 4
415 %0 = bitcast i32* %D to i8*
416 %bf = load i8, i8* %0, align 4
417 %bf.clear = and i8 %bf, -3
418 store i8 %bf.clear, i8* %0, align 4
419 %bf1 = load i8, i8* %0, align 4
420 %bf.clear1 = and i8 %bf1, 1
421 store i8 %bf.clear1, i8* %0, align 4
422 ret i32 0
423 }
424
425 ; Check that we do not hoist loads past calls within a same basic block.
426 ; CHECK-LABEL: @noHoistInSingleBBWithCall
427 ; CHECK: load
428 ; CHECK: call
429 ; CHECK: load
430 declare void @foo()
431 define i32 @noHoistInSingleBBWithCall() {
432 entry:
433 %D = alloca i32, align 4
434 %0 = bitcast i32* %D to i8*
435 %bf = load i8, i8* %0, align 4
436 %bf.clear = and i8 %bf, -3
437 call void @foo()
438 %bf1 = load i8, i8* %0, align 4
439 %bf.clear1 = and i8 %bf1, 1
440 ret i32 0
441 }
442
443 ; Check that we do not hoist loads past stores in any branch of a diamond.
444 ; CHECK-LABEL: @noHoistInDiamondWithOneStore1
445 ; CHECK: fdiv
446 ; CHECK: fcmp
447 ; CHECK: br
448 define float @noHoistInDiamondWithOneStore1(float %d, float* %min, float* %max, float* %a) {
449 entry:
450 %div = fdiv float 1.000000e+00, %d
451 %cmp = fcmp oge float %div, 0.000000e+00
452 br i1 %cmp, label %if.then, label %if.else
453
454 if.then: ; preds = %entry
455 store float 0.000000e+00, float* @GlobalVar
456 %0 = load float, float* %min, align 4
457 %1 = load float, float* %a, align 4
458 %sub = fsub float %0, %1
459 %mul = fmul float %sub, %div
460 %2 = load float, float* %max, align 4
461 %sub1 = fsub float %2, %1
462 %mul2 = fmul float %sub1, %div
463 br label %if.end
464
465 if.else: ; preds = %entry
466 ; There are no side effects on the if.else branch.
467 %3 = load float, float* %max, align 4
468 %4 = load float, float* %a, align 4
469 %sub3 = fsub float %3, %4
470 %mul4 = fmul float %sub3, %div
471 %5 = load float, float* %min, align 4
472 %sub5 = fsub float %5, %4
473 %mul6 = fmul float %sub5, %div
474 br label %if.end
475
476 if.end: ; preds = %if.else, %if.then
477 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
478 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
479
480 %6 = load float, float* %max, align 4
481 %7 = load float, float* %a, align 4
482 %sub6 = fsub float %6, %7
483 %mul7 = fmul float %sub6, %div
484 %8 = load float, float* %min, align 4
485 %sub8 = fsub float %8, %7
486 %mul9 = fmul float %sub8, %div
487
488 %add = fadd float %tmax.0, %tmin.0
489 ret float %add
490 }
491
492 ; Check that we do not hoist loads past a store in any branch of a diamond.
493 ; CHECK-LABEL: @noHoistInDiamondWithOneStore2
494 ; CHECK: fdiv
495 ; CHECK: fcmp
496 ; CHECK: br
497 define float @noHoistInDiamondWithOneStore2(float %d, float* %min, float* %max, float* %a) {
498 entry:
499 %div = fdiv float 1.000000e+00, %d
500 %cmp = fcmp oge float %div, 0.000000e+00
501 br i1 %cmp, label %if.then, label %if.else
502
503 if.then: ; preds = %entry
504 ; There are no side effects on the if.then branch.
505 %0 = load float, float* %min, align 4
506 %1 = load float, float* %a, align 4
507 %sub = fsub float %0, %1
508 %mul = fmul float %sub, %div
509 %2 = load float, float* %max, align 4
510 %sub1 = fsub float %2, %1
511 %mul2 = fmul float %sub1, %div
512 br label %if.end
513
514 if.else: ; preds = %entry
515 store float 0.000000e+00, float* @GlobalVar
516 %3 = load float, float* %max, align 4
517 %4 = load float, float* %a, align 4
518 %sub3 = fsub float %3, %4
519 %mul4 = fmul float %sub3, %div
520 %5 = load float, float* %min, align 4
521 %sub5 = fsub float %5, %4
522 %mul6 = fmul float %sub5, %div
523 br label %if.end
524
525 if.end: ; preds = %if.else, %if.then
526 %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
527 %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
528
529 %6 = load float, float* %max, align 4
530 %7 = load float, float* %a, align 4
531 %sub6 = fsub float %6, %7
532 %mul7 = fmul float %sub6, %div
533 %8 = load float, float* %min, align 4
534 %sub8 = fsub float %8, %7
535 %mul9 = fmul float %sub8, %div
536
537 %add = fadd float %tmax.0, %tmin.0
538 ret float %add
539 }
540
541 ; Check that we do not hoist loads outside a loop containing stores.
542 ; CHECK-LABEL: @noHoistInLoopsWithStores
543 ; CHECK: fdiv
544 ; CHECK: fcmp
545 ; CHECK: br
546 define float @noHoistInLoopsWithStores(float %d, float* %min, float* %max, float* %a) {
547 entry:
548 %div = fdiv float 1.000000e+00, %d
549 %cmp = fcmp oge float %div, 0.000000e+00
550 br i1 %cmp, label %do.body, label %if.else
551
552 do.body:
553 %0 = load float, float* %min, align 4
554 %1 = load float, float* %a, align 4
555
556 ; It is unsafe to hoist the loads outside the loop because of the store.
557 store float 0.000000e+00, float* @GlobalVar
558
559 %sub = fsub float %0, %1
560 %mul = fmul float %sub, %div
561 %2 = load float, float* %max, align 4
562 %sub1 = fsub float %2, %1
563 %mul2 = fmul float %sub1, %div
564 br label %while.cond
565
566 while.cond:
567 %cmp1 = fcmp oge float %mul2, 0.000000e+00
568 br i1 %cmp1, label %if.end, label %do.body
569
570 if.else:
571 %3 = load float, float* %max, align 4
572 %4 = load float, float* %a, align 4
573 %sub3 = fsub float %3, %4
574 %mul4 = fmul float %sub3, %div
575 %5 = load float, float* %min, align 4
576 %sub5 = fsub float %5, %4
577 %mul6 = fmul float %sub5, %div
578 br label %if.end
579
580 if.end:
581 %tmax.0 = phi float [ %mul2, %while.cond ], [ %mul6, %if.else ]
582 %tmin.0 = phi float [ %mul, %while.cond ], [ %mul4, %if.else ]
583
584 %add = fadd float %tmax.0, %tmin.0
585 ret float %add
586 }
587
588 ; Check that we hoist stores: all the instructions from the then branch
589 ; should be hoisted.
590 ; CHECK-LABEL: @hoistStores
591 ; CHECK: zext
592 ; CHECK: trunc
593 ; CHECK: getelementptr
594 ; CHECK: load
595 ; CHECK: getelementptr
596 ; CHECK: store
597 ; CHECK: load
598 ; CHECK: load
599 ; CHECK: zext
600 ; CHECK: add
601 ; CHECK: store
602 ; CHECK: br
603 ; CHECK: if.then
604 ; CHECK: br
605
606 %struct.foo = type { i16* }
607
608 define void @hoistStores(%struct.foo* %s, i32* %coord, i1 zeroext %delta) {
609 entry:
610 %frombool = zext i1 %delta to i8
611 %tobool = trunc i8 %frombool to i1
612 br i1 %tobool, label %if.then, label %if.else
613
614 if.then: ; preds = %entry
615 %p = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
616 %0 = load i16*, i16** %p, align 8
617 %incdec.ptr = getelementptr inbounds i16, i16* %0, i32 1
618 store i16* %incdec.ptr, i16** %p, align 8
619 %1 = load i16, i16* %0, align 2
620 %conv = zext i16 %1 to i32
621 %2 = load i32, i32* %coord, align 4
622 %add = add i32 %2, %conv
623 store i32 %add, i32* %coord, align 4
624 br label %if.end
625
626 if.else: ; preds = %entry
627 %p1 = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
628 %3 = load i16*, i16** %p1, align 8
629 %incdec.ptr2 = getelementptr inbounds i16, i16* %3, i32 1
630 store i16* %incdec.ptr2, i16** %p1, align 8
631 %4 = load i16, i16* %3, align 2
632 %conv3 = zext i16 %4 to i32
633 %5 = load i32, i32* %coord, align 4
634 %add4 = add i32 %5, %conv3
635 store i32 %add4, i32* %coord, align 4
636 %6 = load i16*, i16** %p1, align 8
637 %incdec.ptr6 = getelementptr inbounds i16, i16* %6, i32 1
638 store i16* %incdec.ptr6, i16** %p1, align 8
639 %7 = load i16, i16* %6, align 2
640 %conv7 = zext i16 %7 to i32
641 %shl = shl i32 %conv7, 8
642 %8 = load i32, i32* %coord, align 4
643 %add8 = add i32 %8, %shl
644 store i32 %add8, i32* %coord, align 4
645 br label %if.end
646
647 if.end: ; preds = %if.else, %if.then
648 ret void
649 }