llvm.org GIT mirror llvm / 221779b
[NFC] move some hardware loop checking code to a common place for other using. Differential Revision: https://reviews.llvm.org/D63478 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@363758 91177308-0d34-0410-b5e6-96231b3b80d8 Chen Zheng 28 days ago
9 changed file(s) with 124 addition(s) and 105 deletion(s). Raw diff Collapse all Expand all
2626 #include "llvm/Pass.h"
2727 #include "llvm/Support/AtomicOrdering.h"
2828 #include "llvm/Support/DataTypes.h"
29 #include "llvm/Analysis/LoopInfo.h"
30 #include "llvm/Analysis/ScalarEvolution.h"
31 #include "llvm/IR/Dominators.h"
2932 #include
3033
3134 namespace llvm {
7275 return (Ordering == AtomicOrdering::NotAtomic ||
7376 Ordering == AtomicOrdering::Unordered) && !IsVolatile;
7477 }
78 };
79
80 /// Attributes of a target dependent hardware loop.
81 struct HardwareLoopInfo {
82 HardwareLoopInfo() = delete;
83 HardwareLoopInfo(Loop *L) : L(L) {}
84 Loop *L = nullptr;
85 BasicBlock *ExitBlock = nullptr;
86 BranchInst *ExitBranch = nullptr;
87 const SCEV *ExitCount = nullptr;
88 IntegerType *CountType = nullptr;
89 Value *LoopDecrement = nullptr; // Decrement the loop counter by this
90 // value in every iteration.
91 bool IsNestingLegal = false; // Can a hardware loop be a parent to
92 // another hardware loop?
93 bool CounterInReg = false; // Should loop counter be updated in
94 // the loop via a phi?
95 bool isHardwareLoopCandidate(ScalarEvolution &SE, LoopInfo &LI,
96 DominatorTree &DT, bool ForceNestedLoop = false,
97 bool ForceHardwareLoopPHI = false);
7598 };
7699
77100 /// This pass provides access to the codegen interfaces that are needed
446469 /// target-independent defaults.
447470 void getUnrollingPreferences(Loop *L, ScalarEvolution &,
448471 UnrollingPreferences &UP) const;
449
450 /// Attributes of a target dependent hardware loop.
451 struct HardwareLoopInfo {
452 HardwareLoopInfo() = delete;
453 HardwareLoopInfo(Loop *L) : L(L) { }
454 Loop *L = nullptr;
455 BasicBlock *ExitBlock = nullptr;
456 BranchInst *ExitBranch = nullptr;
457 const SCEV *ExitCount = nullptr;
458 IntegerType *CountType = nullptr;
459 Value *LoopDecrement = nullptr; // Decrement the loop counter by this
460 // value in every iteration.
461 bool IsNestingLegal = false; // Can a hardware loop be a parent to
462 // another hardware loop?
463 bool CounterInReg = false; // Should loop counter be updated in
464 // the loop via a phi?
465 };
466472
467473 /// Query the target whether it would be profitable to convert the given loop
468474 /// into a hardware loop.
192192 bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
193193 AssumptionCache &AC,
194194 TargetLibraryInfo *LibInfo,
195 TTI::HardwareLoopInfo &HWLoopInfo) {
195 HardwareLoopInfo &HWLoopInfo) {
196196 return false;
197197 }
198198
494494 bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
495495 AssumptionCache &AC,
496496 TargetLibraryInfo *LibInfo,
497 TTI::HardwareLoopInfo &HWLoopInfo) {
497 HardwareLoopInfo &HWLoopInfo) {
498498 return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
499499 }
500500
3737 explicit NoTTIImpl(const DataLayout &DL)
3838 : TargetTransformInfoImplCRTPBase(DL) {}
3939 };
40 }
41
42 bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
43 LoopInfo &LI, DominatorTree &DT,
44 bool ForceNestedLoop,
45 bool ForceHardwareLoopPHI) {
46 SmallVector ExitingBlocks;
47 L->getExitingBlocks(ExitingBlocks);
48
49 for (SmallVectorImpl::iterator I = ExitingBlocks.begin(),
50 IE = ExitingBlocks.end();
51 I != IE; ++I) {
52 BasicBlock *BB = *I;
53
54 // If we pass the updated counter back through a phi, we need to know
55 // which latch the updated value will be coming from.
56 if (!L->isLoopLatch(BB)) {
57 if (ForceHardwareLoopPHI || CounterInReg)
58 continue;
59 }
60
61 const SCEV *EC = SE.getExitCount(L, BB);
62 if (isa(EC))
63 continue;
64 if (const SCEVConstant *ConstEC = dyn_cast(EC)) {
65 if (ConstEC->getValue()->isZero())
66 continue;
67 } else if (!SE.isLoopInvariant(EC, L))
68 continue;
69
70 if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth())
71 continue;
72
73 // If this exiting block is contained in a nested loop, it is not eligible
74 // for insertion of the branch-and-decrement since the inner loop would
75 // end up messing up the value in the CTR.
76 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
77 continue;
78
79 // We now have a loop-invariant count of loop iterations (which is not the
80 // constant zero) for which we know that this loop will not exit via this
81 // existing block.
82
83 // We need to make sure that this block will run on every loop iteration.
84 // For this to be true, we must dominate all blocks with backedges. Such
85 // blocks are in-loop predecessors to the header block.
86 bool NotAlways = false;
87 for (pred_iterator PI = pred_begin(L->getHeader()),
88 PIE = pred_end(L->getHeader());
89 PI != PIE; ++PI) {
90 if (!L->contains(*PI))
91 continue;
92
93 if (!DT.dominates(*I, *PI)) {
94 NotAlways = true;
95 break;
96 }
97 }
98
99 if (NotAlways)
100 continue;
101
102 // Make sure this blocks ends with a conditional branch.
103 Instruction *TI = BB->getTerminator();
104 if (!TI)
105 continue;
106
107 if (BranchInst *BI = dyn_cast(TI)) {
108 if (!BI->isConditional())
109 continue;
110
111 ExitBranch = BI;
112 } else
113 continue;
114
115 // Note that this block may not be the loop latch block, even if the loop
116 // has a latch block.
117 ExitBlock = *I;
118 ExitCount = EC;
119 break;
120 }
121
122 if (!ExitBlock)
123 return false;
124 return true;
40125 }
41126
42127 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
100100
101101 // Given that the target believes the loop to be profitable, try to
102102 // convert it.
103 bool TryConvertLoop(TTI::HardwareLoopInfo &HWLoopInfo);
103 bool TryConvertLoop(HardwareLoopInfo &HWLoopInfo);
104104
105105 private:
106106 ScalarEvolution *SE = nullptr;
138138 void UpdateBranch(Value *EltsRem);
139139
140140 public:
141 HardwareLoop(TTI::HardwareLoopInfo &Info, ScalarEvolution &SE,
141 HardwareLoop(HardwareLoopInfo &Info, ScalarEvolution &SE,
142142 const DataLayout &DL) :
143143 SE(SE), DL(DL), L(Info.L), M(L->getHeader()->getModule()),
144144 ExitCount(Info.ExitCount),
204204 if (containsIrreducibleCFG(RPOT, *LI))
205205 return false;
206206
207 TTI::HardwareLoopInfo HWLoopInfo(L);
207 HardwareLoopInfo HWLoopInfo(L);
208208 if (TTI->isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo) ||
209209 ForceHardwareLoops) {
210210
224224 return false;
225225 }
226226
227 bool HardwareLoops::TryConvertLoop(TTI::HardwareLoopInfo &HWLoopInfo) {
227 bool HardwareLoops::TryConvertLoop(HardwareLoopInfo &HWLoopInfo) {
228228
229229 Loop *L = HWLoopInfo.L;
230230 LLVM_DEBUG(dbgs() << "HWLoops: Try to convert profitable loop: " << *L);
231231
232 SmallVector ExitingBlocks;
233 L->getExitingBlocks(ExitingBlocks);
234
235 for (SmallVectorImpl::iterator I = ExitingBlocks.begin(),
236 IE = ExitingBlocks.end(); I != IE; ++I) {
237 BasicBlock *BB = *I;
238
239 // If we pass the updated counter back through a phi, we need to know
240 // which latch the updated value will be coming from.
241 if (!L->isLoopLatch(BB)) {
242 if ((ForceHardwareLoopPHI.getNumOccurrences() && ForceHardwareLoopPHI) ||
243 HWLoopInfo.CounterInReg)
244 continue;
245 }
246
247 const SCEV *EC = SE->getExitCount(L, BB);
248 if (isa(EC))
249 continue;
250 if (const SCEVConstant *ConstEC = dyn_cast(EC)) {
251 if (ConstEC->getValue()->isZero())
252 continue;
253 } else if (!SE->isLoopInvariant(EC, L))
254 continue;
255
256 if (SE->getTypeSizeInBits(EC->getType()) >
257 HWLoopInfo.CountType->getBitWidth())
258 continue;
259
260 // If this exiting block is contained in a nested loop, it is not eligible
261 // for insertion of the branch-and-decrement since the inner loop would
262 // end up messing up the value in the CTR.
263 if (!HWLoopInfo.IsNestingLegal && LI->getLoopFor(BB) != L &&
264 !ForceNestedLoop)
265 continue;
266
267 // We now have a loop-invariant count of loop iterations (which is not the
268 // constant zero) for which we know that this loop will not exit via this
269 // existing block.
270
271 // We need to make sure that this block will run on every loop iteration.
272 // For this to be true, we must dominate all blocks with backedges. Such
273 // blocks are in-loop predecessors to the header block.
274 bool NotAlways = false;
275 for (pred_iterator PI = pred_begin(L->getHeader()),
276 PIE = pred_end(L->getHeader()); PI != PIE; ++PI) {
277 if (!L->contains(*PI))
278 continue;
279
280 if (!DT->dominates(*I, *PI)) {
281 NotAlways = true;
282 break;
283 }
284 }
285
286 if (NotAlways)
287 continue;
288
289 // Make sure this blocks ends with a conditional branch.
290 Instruction *TI = BB->getTerminator();
291 if (!TI)
292 continue;
293
294 if (BranchInst *BI = dyn_cast(TI)) {
295 if (!BI->isConditional())
296 continue;
297
298 HWLoopInfo.ExitBranch = BI;
299 } else
300 continue;
301
302 // Note that this block may not be the loop latch block, even if the loop
303 // has a latch block.
304 HWLoopInfo.ExitBlock = *I;
305 HWLoopInfo.ExitCount = EC;
306 break;
307 }
308
309 if (!HWLoopInfo.ExitBlock)
232 if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT, ForceNestedLoop,
233 ForceHardwareLoopPHI))
310234 return false;
235
236 assert(
237 (HWLoopInfo.ExitBlock && HWLoopInfo.ExitBranch && HWLoopInfo.ExitCount) &&
238 "Hardware Loop must have set exit info.");
311239
312240 BasicBlock *Preheader = L->getLoopPreheader();
313241
695695 bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
696696 AssumptionCache &AC,
697697 TargetLibraryInfo *LibInfo,
698 TTI::HardwareLoopInfo &HWLoopInfo) {
698 HardwareLoopInfo &HWLoopInfo) {
699699 // Low-overhead branches are only supported in the 'low-overhead branch'
700700 // extension of v8.1-m.
701701 if (!ST->hasLOB() || DisableLowOverheadLoops)
183183 bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
184184 AssumptionCache &AC,
185185 TargetLibraryInfo *LibInfo,
186 TTI::HardwareLoopInfo &HWLoopInfo);
186 HardwareLoopInfo &HWLoopInfo);
187187
188188 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
189189 TTI::UnrollingPreferences &UP);
491491 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
492492 AssumptionCache &AC,
493493 TargetLibraryInfo *LibInfo,
494 TTI::HardwareLoopInfo &HWLoopInfo) {
494 HardwareLoopInfo &HWLoopInfo) {
495495 const PPCTargetMachine &TM = ST->getTargetMachine();
496496 TargetSchedModel SchedModel;
497497 SchedModel.init(ST);
5555 bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
5656 AssumptionCache &AC,
5757 TargetLibraryInfo *LibInfo,
58 TTI::HardwareLoopInfo &HWLoopInfo);
58 HardwareLoopInfo &HWLoopInfo);
5959 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
6060 TTI::UnrollingPreferences &UP);
6161