llvm.org GIT mirror llvm / 45baf6b
Fix the API usage in loop probability heuristics. It was incorrectly classifying many edges as exiting which were in fact not. These mainly formed edges into sub-loops. It was also not correctly classifying all returning edges out of loops as leaving the loop. With this match most of the loop heuristics are more rational. Several serious regressions on loop-intesive benchmarks like perlbench's loop tests when built with -enable-block-placement are fixed by these updated heuristics. Unfortunately they in turn uncover some other regressions. There are still several improvemenst that should be made to loop heuristics including trip-count, and early back-edge management. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@142917 91177308-0d34-0410-b5e6-96231b3b80d8 Chandler Carruth 8 years ago
2 changed file(s) with 373 addition(s) and 15 deletion(s). Raw diff Collapse all Expand all
215215 // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges
216216 // as taken, exiting edges as not-taken.
217217 bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
218 uint32_t numSuccs = BB->getTerminator()->getNumSuccessors();
219
220218 Loop *L = LI->getLoopFor(BB);
221219 if (!L)
222220 return false;
225223 SmallPtrSet ExitingEdges;
226224 SmallPtrSet InEdges; // Edges from header to the loop.
227225
228 bool isHeader = BB == L->getHeader();
229
230226 for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
231 BasicBlock *Succ = *I;
232 Loop *SuccL = LI->getLoopFor(Succ);
233 if (SuccL != L)
234 ExitingEdges.insert(Succ);
235 else if (Succ == L->getHeader())
236 BackEdges.insert(Succ);
237 else if (isHeader)
238 InEdges.insert(Succ);
227 if (!L->contains(*I))
228 ExitingEdges.insert(*I);
229 else if (L->getHeader() == *I)
230 BackEdges.insert(*I);
231 else
232 InEdges.insert(*I);
239233 }
240234
241235 if (uint32_t numBackEdges = BackEdges.size()) {
262256 }
263257 }
264258
265 uint32_t numExitingEdges = ExitingEdges.size();
266 if (uint32_t numNonExitingEdges = numSuccs - numExitingEdges) {
267 uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges;
259 if (uint32_t numExitingEdges = ExitingEdges.size()) {
260 uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numExitingEdges;
268261 if (exitWeight < MIN_WEIGHT)
269262 exitWeight = MIN_WEIGHT;
270263
0 ; Test the static branch probability heuristics for no-return functions.
1 ; RUN: opt < %s -analyze -branch-prob | FileCheck %s
2
3 declare void @g1()
4 declare void @g2()
5 declare void @g3()
6 declare void @g4()
7
8 define void @test1(i32 %a, i32 %b) {
9 entry:
10 br label %do.body
11 ; CHECK: edge entry -> do.body probability is 16 / 16 = 100%
12
13 do.body:
14 %i.0 = phi i32 [ 0, %entry ], [ %inc3, %do.end ]
15 call void @g1()
16 br label %do.body1
17 ; CHECK: edge do.body -> do.body1 probability is 124 / 124 = 100%
18
19 do.body1:
20 %j.0 = phi i32 [ 0, %do.body ], [ %inc, %do.body1 ]
21 call void @g2()
22 %inc = add nsw i32 %j.0, 1
23 %cmp = icmp slt i32 %inc, %b
24 br i1 %cmp, label %do.body1, label %do.end
25 ; CHECK: edge do.body1 -> do.body1 probability is 124 / 128
26 ; CHECK: edge do.body1 -> do.end probability is 4 / 128
27
28 do.end:
29 call void @g3()
30 %inc3 = add nsw i32 %i.0, 1
31 %cmp4 = icmp slt i32 %inc3, %a
32 br i1 %cmp4, label %do.body, label %do.end5
33 ; CHECK: edge do.end -> do.body probability is 124 / 128
34 ; CHECK: edge do.end -> do.end5 probability is 4 / 128
35
36 do.end5:
37 call void @g4()
38 ret void
39 }
40
41 define void @test2(i32 %a, i32 %b) {
42 entry:
43 %cmp9 = icmp sgt i32 %a, 0
44 br i1 %cmp9, label %for.body.lr.ph, label %for.end6
45 ; CHECK: edge entry -> for.body.lr.ph probability is 20 / 32
46 ; CHECK: edge entry -> for.end6 probability is 12 / 32
47
48 for.body.lr.ph:
49 %cmp27 = icmp sgt i32 %b, 0
50 br label %for.body
51 ; CHECK: edge for.body.lr.ph -> for.body probability is 16 / 16 = 100%
52
53 for.body:
54 %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc5, %for.end ]
55 call void @g1()
56 br i1 %cmp27, label %for.body3, label %for.end
57 ; CHECK: edge for.body -> for.body3 probability is 62 / 124 = 50%
58 ; CHECK: edge for.body -> for.end probability is 62 / 124 = 50%
59
60 for.body3:
61 %j.08 = phi i32 [ %inc, %for.body3 ], [ 0, %for.body ]
62 call void @g2()
63 %inc = add nsw i32 %j.08, 1
64 %exitcond = icmp eq i32 %inc, %b
65 br i1 %exitcond, label %for.end, label %for.body3
66 ; CHECK: edge for.body3 -> for.end probability is 4 / 128
67 ; CHECK: edge for.body3 -> for.body3 probability is 124 / 128
68
69 for.end:
70 call void @g3()
71 %inc5 = add nsw i32 %i.010, 1
72 %exitcond11 = icmp eq i32 %inc5, %a
73 br i1 %exitcond11, label %for.end6, label %for.body
74 ; CHECK: edge for.end -> for.end6 probability is 4 / 128
75 ; CHECK: edge for.end -> for.body probability is 124 / 128
76
77 for.end6:
78 call void @g4()
79 ret void
80 }
81
82 define void @test3(i32 %a, i32 %b, i32* %c) {
83 entry:
84 br label %do.body
85 ; CHECK: edge entry -> do.body probability is 16 / 16 = 100%
86
87 do.body:
88 %i.0 = phi i32 [ 0, %entry ], [ %inc4, %if.end ]
89 call void @g1()
90 %0 = load i32* %c, align 4
91 %cmp = icmp slt i32 %0, 42
92 br i1 %cmp, label %do.body1, label %if.end
93 ; CHECK: edge do.body -> do.body1 probability is 62 / 124 = 50%
94 ; CHECK: edge do.body -> if.end probability is 62 / 124 = 50%
95
96 do.body1:
97 %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ]
98 call void @g2()
99 %inc = add nsw i32 %j.0, 1
100 %cmp2 = icmp slt i32 %inc, %b
101 br i1 %cmp2, label %do.body1, label %if.end
102 ; CHECK: edge do.body1 -> do.body1 probability is 124 / 128
103 ; CHECK: edge do.body1 -> if.end probability is 4 / 128
104
105 if.end:
106 call void @g3()
107 %inc4 = add nsw i32 %i.0, 1
108 %cmp5 = icmp slt i32 %inc4, %a
109 br i1 %cmp5, label %do.body, label %do.end6
110 ; CHECK: edge if.end -> do.body probability is 124 / 128
111 ; CHECK: edge if.end -> do.end6 probability is 4 / 128
112
113 do.end6:
114 call void @g4()
115 ret void
116 }
117
118 define void @test4(i32 %a, i32 %b, i32* %c) {
119 entry:
120 br label %do.body
121 ; CHECK: edge entry -> do.body probability is 16 / 16 = 100%
122
123 do.body:
124 %i.0 = phi i32 [ 0, %entry ], [ %inc4, %do.end ]
125 call void @g1()
126 %0 = load i32* %c, align 4
127 %cmp = icmp slt i32 %0, 42
128 br i1 %cmp, label %return, label %do.body1
129 ; CHECK: edge do.body -> return probability is 4 / 128
130 ; CHECK: edge do.body -> do.body1 probability is 124 / 128
131
132 do.body1:
133 %j.0 = phi i32 [ %inc, %do.body1 ], [ 0, %do.body ]
134 call void @g2()
135 %inc = add nsw i32 %j.0, 1
136 %cmp2 = icmp slt i32 %inc, %b
137 br i1 %cmp2, label %do.body1, label %do.end
138 ; CHECK: edge do.body1 -> do.body1 probability is 124 / 128
139 ; CHECK: edge do.body1 -> do.end probability is 4 / 128
140
141 do.end:
142 call void @g3()
143 %inc4 = add nsw i32 %i.0, 1
144 %cmp5 = icmp slt i32 %inc4, %a
145 br i1 %cmp5, label %do.body, label %do.end6
146 ; CHECK: edge do.end -> do.body probability is 124 / 128
147 ; CHECK: edge do.end -> do.end6 probability is 4 / 128
148
149 do.end6:
150 call void @g4()
151 br label %return
152 ; CHECK: edge do.end6 -> return probability is 16 / 16 = 100%
153
154 return:
155 ret void
156 }
157
158 define void @test5(i32 %a, i32 %b, i32* %c) {
159 entry:
160 br label %do.body
161 ; CHECK: edge entry -> do.body probability is 16 / 16 = 100%
162
163 do.body:
164 %i.0 = phi i32 [ 0, %entry ], [ %inc4, %do.end ]
165 call void @g1()
166 br label %do.body1
167 ; CHECK: edge do.body -> do.body1 probability is 124 / 124 = 100%
168
169 do.body1:
170 %j.0 = phi i32 [ 0, %do.body ], [ %inc, %if.end ]
171 %0 = load i32* %c, align 4
172 %cmp = icmp slt i32 %0, 42
173 br i1 %cmp, label %return, label %if.end
174 ; CHECK: edge do.body1 -> return probability is 4 / 128
175 ; CHECK: edge do.body1 -> if.end probability is 124 / 128
176
177 if.end:
178 call void @g2()
179 %inc = add nsw i32 %j.0, 1
180 %cmp2 = icmp slt i32 %inc, %b
181 br i1 %cmp2, label %do.body1, label %do.end
182 ; CHECK: edge if.end -> do.body1 probability is 124 / 128
183 ; CHECK: edge if.end -> do.end probability is 4 / 128
184
185 do.end:
186 call void @g3()
187 %inc4 = add nsw i32 %i.0, 1
188 %cmp5 = icmp slt i32 %inc4, %a
189 br i1 %cmp5, label %do.body, label %do.end6
190 ; CHECK: edge do.end -> do.body probability is 124 / 128
191 ; CHECK: edge do.end -> do.end6 probability is 4 / 128
192
193 do.end6:
194 call void @g4()
195 br label %return
196 ; CHECK: edge do.end6 -> return probability is 16 / 16 = 100%
197
198 return:
199 ret void
200 }
201
202 define void @test6(i32 %a, i32 %b, i32* %c) {
203 entry:
204 br label %do.body
205 ; CHECK: edge entry -> do.body probability is 16 / 16 = 100%
206
207 do.body:
208 %i.0 = phi i32 [ 0, %entry ], [ %inc4, %do.end ]
209 call void @g1()
210 br label %do.body1
211 ; CHECK: edge do.body -> do.body1 probability is 124 / 124 = 100%
212
213 do.body1:
214 %j.0 = phi i32 [ 0, %do.body ], [ %inc, %do.cond ]
215 call void @g2()
216 %0 = load i32* %c, align 4
217 %cmp = icmp slt i32 %0, 42
218 br i1 %cmp, label %return, label %do.cond
219 ; CHECK: edge do.body1 -> return probability is 4 / 128
220 ; CHECK: edge do.body1 -> do.cond probability is 124 / 128
221
222 do.cond:
223 %inc = add nsw i32 %j.0, 1
224 %cmp2 = icmp slt i32 %inc, %b
225 br i1 %cmp2, label %do.body1, label %do.end
226 ; CHECK: edge do.cond -> do.body1 probability is 124 / 128
227 ; CHECK: edge do.cond -> do.end probability is 4 / 128
228
229 do.end:
230 call void @g3()
231 %inc4 = add nsw i32 %i.0, 1
232 %cmp5 = icmp slt i32 %inc4, %a
233 br i1 %cmp5, label %do.body, label %do.end6
234 ; CHECK: edge do.end -> do.body probability is 124 / 128
235 ; CHECK: edge do.end -> do.end6 probability is 4 / 128
236
237 do.end6:
238 call void @g4()
239 br label %return
240 ; CHECK: edge do.end6 -> return probability is 16 / 16 = 100%
241
242 return:
243 ret void
244 }
245
246 define void @test7(i32 %a, i32 %b, i32* %c) {
247 entry:
248 %cmp10 = icmp sgt i32 %a, 0
249 br i1 %cmp10, label %for.body.lr.ph, label %for.end7
250 ; CHECK: edge entry -> for.body.lr.ph probability is 20 / 32
251 ; CHECK: edge entry -> for.end7 probability is 12 / 32
252
253 for.body.lr.ph:
254 %cmp38 = icmp sgt i32 %b, 0
255 br label %for.body
256 ; CHECK: edge for.body.lr.ph -> for.body probability is 16 / 16 = 100%
257
258 for.body:
259 %i.011 = phi i32 [ 0, %for.body.lr.ph ], [ %inc6, %for.inc5 ]
260 %0 = load i32* %c, align 4
261 %cmp1 = icmp eq i32 %0, %i.011
262 br i1 %cmp1, label %for.inc5, label %if.end
263 ; CHECK: edge for.body -> for.inc5 probability is 62 / 124 = 50%
264 ; CHECK: edge for.body -> if.end probability is 62 / 124 = 50%
265
266 if.end:
267 call void @g1()
268 br i1 %cmp38, label %for.body4, label %for.end
269 ; CHECK: edge if.end -> for.body4 probability is 62 / 124 = 50%
270 ; CHECK: edge if.end -> for.end probability is 62 / 124 = 50%
271
272 for.body4:
273 %j.09 = phi i32 [ %inc, %for.body4 ], [ 0, %if.end ]
274 call void @g2()
275 %inc = add nsw i32 %j.09, 1
276 %exitcond = icmp eq i32 %inc, %b
277 br i1 %exitcond, label %for.end, label %for.body4
278 ; CHECK: edge for.body4 -> for.end probability is 4 / 128
279 ; CHECK: edge for.body4 -> for.body4 probability is 124 / 128
280
281 for.end:
282 call void @g3()
283 br label %for.inc5
284 ; CHECK: edge for.end -> for.inc5 probability is 124 / 124 = 100%
285
286 for.inc5:
287 %inc6 = add nsw i32 %i.011, 1
288 %exitcond12 = icmp eq i32 %inc6, %a
289 br i1 %exitcond12, label %for.end7, label %for.body
290 ; CHECK: edge for.inc5 -> for.end7 probability is 4 / 128
291 ; CHECK: edge for.inc5 -> for.body probability is 124 / 128
292
293 for.end7:
294 call void @g4()
295 ret void
296 }
297
298 define void @test8(i32 %a, i32 %b, i32* %c) {
299 entry:
300 %cmp18 = icmp sgt i32 %a, 0
301 br i1 %cmp18, label %for.body.lr.ph, label %for.end15
302 ; CHECK: edge entry -> for.body.lr.ph probability is 20 / 32
303 ; CHECK: edge entry -> for.end15 probability is 12 / 32
304
305 for.body.lr.ph:
306 %cmp216 = icmp sgt i32 %b, 0
307 %arrayidx5 = getelementptr inbounds i32* %c, i64 1
308 %arrayidx9 = getelementptr inbounds i32* %c, i64 2
309 br label %for.body
310 ; CHECK: edge for.body.lr.ph -> for.body probability is 16 / 16 = 100%
311
312 for.body:
313 %i.019 = phi i32 [ 0, %for.body.lr.ph ], [ %inc14, %for.end ]
314 call void @g1()
315 br i1 %cmp216, label %for.body3, label %for.end
316 ; CHECK: edge for.body -> for.body3 probability is 62 / 124 = 50%
317 ; CHECK: edge for.body -> for.end probability is 62 / 124 = 50%
318
319 for.body3:
320 %j.017 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ]
321 %0 = load i32* %c, align 4
322 %cmp4 = icmp eq i32 %0, %j.017
323 br i1 %cmp4, label %for.inc, label %if.end
324 ; CHECK: edge for.body3 -> for.inc probability is 62 / 124 = 50%
325 ; CHECK: edge for.body3 -> if.end probability is 62 / 124 = 50%
326
327 if.end:
328 %1 = load i32* %arrayidx5, align 4
329 %cmp6 = icmp eq i32 %1, %j.017
330 br i1 %cmp6, label %for.inc, label %if.end8
331 ; CHECK: edge if.end -> for.inc probability is 62 / 124 = 50%
332 ; CHECK: edge if.end -> if.end8 probability is 62 / 124 = 50%
333
334 if.end8:
335 %2 = load i32* %arrayidx9, align 4
336 %cmp10 = icmp eq i32 %2, %j.017
337 br i1 %cmp10, label %for.inc, label %if.end12
338 ; CHECK: edge if.end8 -> for.inc probability is 62 / 124 = 50%
339 ; CHECK: edge if.end8 -> if.end12 probability is 62 / 124 = 50%
340
341 if.end12:
342 call void @g2()
343 br label %for.inc
344 ; CHECK: edge if.end12 -> for.inc probability is 124 / 124 = 100%
345
346 for.inc:
347 %inc = add nsw i32 %j.017, 1
348 %exitcond = icmp eq i32 %inc, %b
349 br i1 %exitcond, label %for.end, label %for.body3
350 ; CHECK: edge for.inc -> for.end probability is 4 / 128
351 ; CHECK: edge for.inc -> for.body3 probability is 124 / 128
352
353 for.end:
354 call void @g3()
355 %inc14 = add nsw i32 %i.019, 1
356 %exitcond20 = icmp eq i32 %inc14, %a
357 br i1 %exitcond20, label %for.end15, label %for.body
358 ; CHECK: edge for.end -> for.end15 probability is 4 / 128
359 ; CHECK: edge for.end -> for.body probability is 124 / 128
360
361 for.end15:
362 call void @g4()
363 ret void
364 }