llvm.org GIT mirror llvm / 28a739b
SLPVectorizer: Bring back the insertelement patch (r205965) with fixes When can't assume a vectorized tree is rooted in an instruction. The IRBuilder could have constant folded it. When we rebuild the build_vector (the series of InsertElement instructions) use the last original InsertElement instruction. The vectorized tree root is guaranteed to be before it. Also, we can't assume that the n-th InsertElement inserts the n-th element into a vector. This reverts r207746 which reverted the revert of the revert of r205018 or so. Fixes the test case in PR19621. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@207939 91177308-0d34-0410-b5e6-96231b3b80d8 Arnold Schwaighofer 5 years ago
3 changed file(s) with 211 addition(s) and 30 deletion(s). Raw diff Collapse all Expand all
3030 #include "llvm/IR/Instructions.h"
3131 #include "llvm/IR/IntrinsicInst.h"
3232 #include "llvm/IR/Module.h"
33 #include "llvm/IR/NoFolder.h"
3334 #include "llvm/IR/Type.h"
3435 #include "llvm/IR/Value.h"
3536 #include "llvm/IR/Verifier.h"
356357 /// A negative number means that this is profitable.
357358 int getTreeCost();
358359
359 /// Construct a vectorizable tree that starts at \p Roots and is possibly
360 /// used by a reduction of \p RdxOps.
361 void buildTree(ArrayRef Roots, ValueSet *RdxOps = 0);
360 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
361 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
362 void buildTree(ArrayRef Roots,
363 ArrayRef UserIgnoreLst = None);
362364
363365 /// Clear the internal data structures that are created by 'buildTree'.
364366 void deleteTree() {
365 RdxOps = 0;
366367 VectorizableTree.clear();
367368 ScalarToTreeEntry.clear();
368369 MustGather.clear();
525526 return I.first->second;
526527 }
527528
528 /// Reduction operators.
529 ValueSet *RdxOps;
529 /// List of users to ignore during scheduling and that don't need extracting.
530 ArrayRef UserIgnoreList;
530531
531532 // Analysis and block reference.
532533 Function *F;
541542 IRBuilder<> Builder;
542543 };
543544
544 void BoUpSLP::buildTree(ArrayRef Roots, ValueSet *Rdx) {
545 void BoUpSLP::buildTree(ArrayRef Roots,
546 ArrayRef UserIgnoreLst) {
545547 deleteTree();
546 RdxOps = Rdx;
548 UserIgnoreList = UserIgnoreLst;
547549 if (!getSameType(Roots))
548550 return;
549551 buildTree_rec(Roots, 0);
575577 if (!UserInst)
576578 continue;
577579
578 // Ignore uses that are part of the reduction.
579 if (Rdx && std::find(Rdx->begin(), Rdx->end(), UserInst) != Rdx->end())
580 // Ignore users in the user ignore list.
581 if (std::find(UserIgnoreList.begin(), UserIgnoreList.end(), UserInst) !=
582 UserIgnoreList.end())
580583 continue;
581584
582585 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
707710 continue;
708711 }
709712
710 // This user is part of the reduction.
711 if (RdxOps && RdxOps->count(UI))
713 // Ignore users in the user ignore list.
714 if (std::find(UserIgnoreList.begin(), UserIgnoreList.end(), UI) !=
715 UserIgnoreList.end())
712716 continue;
713717
714718 // Make sure that we can schedule this unknown user.
17461750 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
17471751
17481752 assert((ScalarToTreeEntry.count(U) ||
1749 // It is legal to replace the reduction users by undef.
1750 (RdxOps && RdxOps->count(U))) &&
1753 // It is legal to replace users in the ignorelist by undef.
1754 (std::find(UserIgnoreList.begin(), UserIgnoreList.end(), U) !=
1755 UserIgnoreList.end())) &&
17511756 "Replacing out-of-tree value with undef");
17521757 }
17531758 #endif
19531958 bool tryToVectorizePair(Value *A, Value *B, BoUpSLP &R);
19541959
19551960 /// \brief Try to vectorize a list of operands.
1961 /// \@param BuildVector A list of users to ignore for the purpose of
1962 /// scheduling and that don't need extracting.
19561963 /// \returns true if a value was vectorized.
1957 bool tryToVectorizeList(ArrayRef VL, BoUpSLP &R);
1964 bool tryToVectorizeList(ArrayRef VL, BoUpSLP &R,
1965 ArrayRef BuildVector = None);
19581966
19591967 /// \brief Try to vectorize a chain that may start at the operands of \V;
19601968 bool tryToVectorize(BinaryOperator *V, BoUpSLP &R);
21272135 return tryToVectorizeList(VL, R);
21282136 }
21292137
2130 bool SLPVectorizer::tryToVectorizeList(ArrayRef VL, BoUpSLP &R) {
2138 bool SLPVectorizer::tryToVectorizeList(ArrayRef VL, BoUpSLP &R,
2139 ArrayRef BuildVector) {
21312140 if (VL.size() < 2)
21322141 return false;
21332142
21772186 << "\n");
21782187 ArrayRef Ops = VL.slice(i, OpsWidth);
21792188
2180 R.buildTree(Ops);
2189 ArrayRef BuildVectorSlice;
2190 if (!BuildVector.empty())
2191 BuildVectorSlice = BuildVector.slice(i, OpsWidth);
2192
2193 R.buildTree(Ops, BuildVectorSlice);
21812194 int Cost = R.getTreeCost();
21822195
21832196 if (Cost < -SLPCostThreshold) {
21842197 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
2185 R.vectorizeTree();
2186
2198 Value *VectorizedRoot = R.vectorizeTree();
2199
2200 // Reconstruct the build vector by extracting the vectorized root. This
2201 // way we handle the case where some elements of the vector are undefined.
2202 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
2203 if (!BuildVectorSlice.empty()) {
2204 // The insert point is the last build vector instruction. The vectorized
2205 // root will precede it. This guarantees that we get an instruction. The
2206 // vectorized tree could have been constant folded.
2207 Instruction *InsertAfter = cast(BuildVectorSlice.back());
2208 unsigned VecIdx = 0;
2209 for (auto &V : BuildVectorSlice) {
2210 IRBuilder Builder(
2211 ++BasicBlock::iterator(InsertAfter));
2212 InsertElementInst *IE = cast(V);
2213 Instruction *Extract = cast(Builder.CreateExtractElement(
2214 VectorizedRoot, Builder.getInt32(VecIdx++)));
2215 IE->setOperand(1, Extract);
2216 IE->removeFromParent();
2217 IE->insertAfter(Extract);
2218 InsertAfter = IE;
2219 }
2220 }
21872221 // Move to the next bundle.
21882222 i += VF - 1;
21892223 Changed = true;
22922326 /// *p =
22932327 ///
22942328 class HorizontalReduction {
2295 SmallPtrSet ReductionOps;
2329 SmallVector ReductionOps;
22962330 SmallVector ReducedVals;
22972331
22982332 BinaryOperator *ReductionRoot;
23862420 // We need to be able to reassociate the adds.
23872421 if (!TreeN->isAssociative())
23882422 return false;
2389 ReductionOps.insert(TreeN);
2423 ReductionOps.push_back(TreeN);
23902424 }
23912425 // Retract.
23922426 Stack.pop_back();
24232457
24242458 for (; i < NumReducedVals - ReduxWidth + 1; i += ReduxWidth) {
24252459 ArrayRef ValsToReduce(&ReducedVals[i], ReduxWidth);
2426 V.buildTree(ValsToReduce, &ReductionOps);
2460 V.buildTree(ValsToReduce, ReductionOps);
24272461
24282462 // Estimate cost.
24292463 int Cost = V.getTreeCost() + getReductionCost(TTI, ReducedVals[i]);
25422576 ///
25432577 /// Returns true if it matches
25442578 ///
2545 static bool findBuildVector(InsertElementInst *IE,
2546 SmallVectorImpl &Ops) {
2547 if (!isa(IE->getOperand(0)))
2579 static bool findBuildVector(InsertElementInst *FirstInsertElem,
2580 SmallVectorImpl &BuildVector,
2581 SmallVectorImpl &BuildVectorOpds) {
2582 if (!isa(FirstInsertElem->getOperand(0)))
25482583 return false;
25492584
2585 InsertElementInst *IE = FirstInsertElem;
25502586 while (true) {
2551 Ops.push_back(IE->getOperand(1));
2587 BuildVector.push_back(IE);
2588 BuildVectorOpds.push_back(IE->getOperand(1));
25522589
25532590 if (IE->use_empty())
25542591 return false;
27192756 }
27202757
27212758 // Try to vectorize trees that start at insertelement instructions.
2722 if (InsertElementInst *IE = dyn_cast(it)) {
2723 SmallVector Ops;
2724 if (!findBuildVector(IE, Ops))
2759 if (InsertElementInst *FirstInsertElem = dyn_cast(it)) {
2760 SmallVector BuildVector;
2761 SmallVector BuildVectorOpds;
2762 if (!findBuildVector(FirstInsertElem, BuildVector, BuildVectorOpds))
27252763 continue;
27262764
2727 if (tryToVectorizeList(Ops, R)) {
2765 // Vectorize starting with the build vector operands ignoring the
2766 // BuildVector instructions for the purpose of scheduling and user
2767 // extraction.
2768 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector)) {
27282769 Changed = true;
27292770 it = BB->begin();
27302771 e = BB->end();
194194 ret <4 x float> %rb
195195 }
196196
197 ; Make sure that vectorization happens even if insertelements operations
198 ; must be rescheduled. The case here is from compiling Julia.
199 define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
200 ; CHECK-LABEL: @reschedule_extract(
201 ; CHECK: %1 = fadd <4 x float> %a, %b
202 %a0 = extractelement <4 x float> %a, i32 0
203 %b0 = extractelement <4 x float> %b, i32 0
204 %c0 = fadd float %a0, %b0
205 %v0 = insertelement <4 x float> undef, float %c0, i32 0
206 %a1 = extractelement <4 x float> %a, i32 1
207 %b1 = extractelement <4 x float> %b, i32 1
208 %c1 = fadd float %a1, %b1
209 %v1 = insertelement <4 x float> %v0, float %c1, i32 1
210 %a2 = extractelement <4 x float> %a, i32 2
211 %b2 = extractelement <4 x float> %b, i32 2
212 %c2 = fadd float %a2, %b2
213 %v2 = insertelement <4 x float> %v1, float %c2, i32 2
214 %a3 = extractelement <4 x float> %a, i32 3
215 %b3 = extractelement <4 x float> %b, i32 3
216 %c3 = fadd float %a3, %b3
217 %v3 = insertelement <4 x float> %v2, float %c3, i32 3
218 ret <4 x float> %v3
219 }
220
197221 ; Check that cost model for vectorization takes credit for
198222 ; instructions that are erased.
199223 define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
218242 ret <4 x float> %v3
219243 }
220244
245 ; Make sure we handle multiple trees that feed one build vector correctly.
246 define <4 x double> @multi_tree(double %w, double %x, double %y, double %z) {
247 entry:
248 %t0 = fadd double %w , 0.000000e+00
249 %t1 = fadd double %x , 1.000000e+00
250 %t2 = fadd double %y , 2.000000e+00
251 %t3 = fadd double %z , 3.000000e+00
252 %t4 = fmul double %t0, 1.000000e+00
253 %i1 = insertelement <4 x double> undef, double %t4, i32 3
254 %t5 = fmul double %t1, 1.000000e+00
255 %i2 = insertelement <4 x double> %i1, double %t5, i32 2
256 %t6 = fmul double %t2, 1.000000e+00
257 %i3 = insertelement <4 x double> %i2, double %t6, i32 1
258 %t7 = fmul double %t3, 1.000000e+00
259 %i4 = insertelement <4 x double> %i3, double %t7, i32 0
260 ret <4 x double> %i4
261 }
262 ; CHECK-LABEL: @multi_tree
263 ; CHECK-DAG: %[[V0:.+]] = insertelement <2 x double> undef, double %w, i32 0
264 ; CHECK-DAG: %[[V1:.+]] = insertelement <2 x double> %[[V0]], double %x, i32 1
265 ; CHECK-DAG: %[[V2:.+]] = fadd <2 x double> %[[V1]],
266 ; CHECK-DAG: %[[V3:.+]] = insertelement <2 x double> undef, double %y, i32 0
267 ; CHECK-DAG: %[[V4:.+]] = insertelement <2 x double> %[[V3]], double %z, i32 1
268 ; CHECK-DAG: %[[V5:.+]] = fadd <2 x double> %[[V4]],
269 ; CHECK-DAG: %[[V6:.+]] = fmul <2 x double> , %[[V2]]
270 ; CHECK-DAG: %[[V7:.+]] = extractelement <2 x double> %[[V6]], i32 0
271 ; CHECK-DAG: %[[I1:.+]] = insertelement <4 x double> undef, double %[[V7]], i32 3
272 ; CHECK-DAG: %[[V8:.+]] = extractelement <2 x double> %[[V6]], i32 1
273 ; CHECK-DAG: %[[I2:.+]] = insertelement <4 x double> %[[I1]], double %[[V8]], i32 2
274 ; CHECK-DAG: %[[V9:.+]] = fmul <2 x double> , %[[V5]]
275 ; CHECK-DAG: %[[V10:.+]] = extractelement <2 x double> %[[V9]], i32 0
276 ; CHECK-DAG: %[[I3:.+]] = insertelement <4 x double> %i2, double %[[V10]], i32 1
277 ; CHECK-DAG: %[[V11:.+]] = extractelement <2 x double> %[[V9]], i32 1
278 ; CHECK-DAG: %[[I4:.+]] = insertelement <4 x double> %i3, double %[[V11]], i32 0
279 ; CHECK: ret <4 x double> %[[I4]]
280
221281 attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
0 ; RUN: opt -slp-vectorizer < %s -S -mtriple="x86_64-grtev3-linux-gnu" -mcpu=corei7-avx | FileCheck %s
1
2 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
3 target triple = "x86_64-grtev3-linux-gnu"
4
5 ; We used to crash on this example because we were building a constant
6 ; expression during vectorization and the vectorizer expects instructions
7 ; as elements of the vectorized tree.
8 ; CHECK-LABEL: @test
9 ; PR19621
10
11 define void @test() {
12 bb279:
13 br label %bb283
14
15 bb283:
16 %Av.sroa.8.0 = phi float [ undef, %bb279 ], [ %tmp315, %exit ]
17 %Av.sroa.5.0 = phi float [ undef, %bb279 ], [ %tmp319, %exit ]
18 %Av.sroa.3.0 = phi float [ undef, %bb279 ], [ %tmp307, %exit ]
19 %Av.sroa.0.0 = phi float [ undef, %bb279 ], [ %tmp317, %exit ]
20 br label %bb284
21
22 bb284:
23 %tmp7.i = fpext float %Av.sroa.3.0 to double
24 %tmp8.i = fsub double %tmp7.i, undef
25 %tmp9.i = fsub double %tmp8.i, undef
26 %tmp17.i = fpext float %Av.sroa.8.0 to double
27 %tmp19.i = fsub double %tmp17.i, undef
28 %tmp20.i = fsub double %tmp19.i, undef
29 br label %bb21.i
30
31 bb21.i:
32 br i1 undef, label %bb22.i, label %exit
33
34 bb22.i:
35 %tmp24.i = fadd double undef, %tmp9.i
36 %tmp26.i = fadd double undef, %tmp20.i
37 br label %bb32.i
38
39 bb32.i:
40 %xs.0.i = phi double [ %tmp24.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
41 %ys.0.i = phi double [ %tmp26.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
42 br i1 undef, label %bb32.i, label %bb21.i
43
44 exit:
45 %tmp303 = fpext float %Av.sroa.0.0 to double
46 %tmp304 = fmul double %tmp303, undef
47 %tmp305 = fadd double undef, %tmp304
48 %tmp306 = fadd double %tmp305, undef
49 %tmp307 = fptrunc double %tmp306 to float
50 %tmp311 = fpext float %Av.sroa.5.0 to double
51 %tmp312 = fmul double %tmp311, 0.000000e+00
52 %tmp313 = fadd double undef, %tmp312
53 %tmp314 = fadd double %tmp313, undef
54 %tmp315 = fptrunc double %tmp314 to float
55 %tmp317 = fptrunc double undef to float
56 %tmp319 = fptrunc double undef to float
57 br label %bb283
58 }
59
60 ; Make sure that we probably handle constant folded vectorized trees. The
61 ; vectorizer starts at the type (%t2, %t3) and wil constant fold the tree.
62 ; The code that handles insertelement instructions must handle this.
63 define <4 x double> @constant_folding() {
64 entry:
65 %t0 = fadd double 1.000000e+00 , 0.000000e+00
66 %t1 = fadd double 1.000000e+00 , 1.000000e+00
67 %t2 = fmul double %t0, 1.000000e+00
68 %i1 = insertelement <4 x double> undef, double %t2, i32 1
69 %t3 = fmul double %t1, 1.000000e+00
70 %i2 = insertelement <4 x double> %i1, double %t3, i32 0
71 ret <4 x double> %i2
72 }
73
74 ; CHECK-LABEL: @constant_folding
75 ; CHECK: %[[V0:.+]] = extractelement <2 x double> , i32 0
76 ; CHECK: %[[V1:.+]] = insertelement <4 x double> undef, double %[[V0]], i32 1
77 ; CHECK: %[[V2:.+]] = extractelement <2 x double> , i32 1
78 ; CHECK: %[[V3:.+]] = insertelement <4 x double> %[[V1]], double %[[V2]], i32 0
79 ; CHECK: ret <4 x double> %[[V3]]