llvm.org GIT mirror llvm / 5172f40
[SLP] Fix regression in broadcasts caused by operand reordering patch D59973. This patch fixes a regression caused by the operand reordering refactoring patch https://reviews.llvm.org/D59973 . The fix changes the strategy to Splat instead of Opcode, if broadcast opportunities are found. Please see the lit test for some examples. Committed on behalf of @vporpo (Vasileios Porpodas) Differential Revision: https://reviews.llvm.org/D62427 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@362613 91177308-0d34-0410-b5e6-96231b3b80d8 Dinar Temirbulatov 3 months ago
2 changed file(s) with 64 addition(s) and 34 deletion(s). Raw diff Collapse all Expand all
916916 /// Clears the data.
917917 void clear() { OpsVec.clear(); }
918918
919 /// \Returns true if there are enough operands identical to \p Op to fill
920 /// the whole vector.
921 /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
922 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
923 bool OpAPO = getData(OpIdx, Lane).APO;
924 for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
925 if (Ln == Lane)
926 continue;
927 // This is set to true if we found a candidate for broadcast at Lane.
928 bool FoundCandidate = false;
929 for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
930 OperandData &Data = getData(OpI, Ln);
931 if (Data.APO != OpAPO || Data.IsUsed)
932 continue;
933 if (Data.V == Op) {
934 FoundCandidate = true;
935 Data.IsUsed = true;
936 break;
937 }
938 }
939 if (!FoundCandidate)
940 return false;
941 }
942 return true;
943 }
944
919945 public:
920946 /// Initialize with all the operands of the instruction vector \p RootVL.
921947 VLOperands(ArrayRef RootVL, const DataLayout &DL,
970996 // side.
971997 if (isa(OpLane0))
972998 ReorderingModes[OpIdx] = ReorderingMode::Load;
973 else if (isa(OpLane0))
974 ReorderingModes[OpIdx] = ReorderingMode::Opcode;
999 else if (isa(OpLane0)) {
1000 // Check if OpLane0 should be broadcast.
1001 if (shouldBroadcast(OpLane0, OpIdx, FirstLane))
1002 ReorderingModes[OpIdx] = ReorderingMode::Splat;
1003 else
1004 ReorderingModes[OpIdx] = ReorderingMode::Opcode;
1005 }
9751006 else if (isa(OpLane0))
9761007 ReorderingModes[OpIdx] = ReorderingMode::Constant;
9771008 else if (isa(OpLane0))
9891020 for (int Pass = 0; Pass != 2; ++Pass) {
9901021 // Skip the second pass if the first pass did not fail.
9911022 bool StrategyFailed = false;
992 // Mark the operand data as free to use for all but the first pass.
993 if (Pass > 0)
994 clearUsed();
1023 // Mark all operand data as free to use.
1024 clearUsed();
9951025 // We keep the original operand order for the FirstLane, so reorder the
9961026 // rest of the lanes. We are visiting the nodes in a circular fashion,
9971027 // using FirstLane as the center point and increasing the radius
66 ; S[2] = %v2 + %v1
77 ; S[3] = %v1 + %v2
88 ;
9 ; TODO: We should broadcast %v1 and %v2
9 ; We broadcast %v1 and %v2
1010 ;
11
1112 define void @bcast_vals(i64 *%A, i64 *%B, i64 *%S) {
1213 ; CHECK-LABEL: @bcast_vals(
1314 ; CHECK-NEXT: entry:
1415 ; CHECK-NEXT: [[A0:%.*]] = load i64, i64* [[A:%.*]], align 8
1516 ; CHECK-NEXT: [[B0:%.*]] = load i64, i64* [[B:%.*]], align 8
16 ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> undef, i64 [[A0]], i32 0
17 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[B0]], i32 1
18 ; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i64> [[TMP1]],
19 ; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <4 x i32>
20 ; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[SHUFFLE]], i32 1
21 ; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> undef, i64 [[TMP3]], i32 0
22 ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[SHUFFLE]], i32 0
23 ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP4]], i64 [[TMP5]], i32 1
24 ; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> undef, <4 x i32>
25 ; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i64> [[SHUFFLE]], [[SHUFFLE1]]
17 ; CHECK-NEXT: [[V1:%.*]] = sub i64 [[A0]], 1
18 ; CHECK-NEXT: [[V2:%.*]] = sub i64 [[B0]], 1
19 ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> undef, i64 [[V1]], i32 0
20 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> [[TMP0]], i64 [[V1]], i32 1
21 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[V1]], i32 2
22 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[V1]], i32 3
23 ; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> undef, i64 [[V2]], i32 0
24 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> [[TMP4]], i64 [[V2]], i32 1
25 ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> [[TMP5]], i64 [[V2]], i32 2
26 ; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i64> [[TMP6]], i64 [[V2]], i32 3
27 ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP3]], [[TMP7]]
2628 ; CHECK-NEXT: [[IDXS0:%.*]] = getelementptr inbounds i64, i64* [[S:%.*]], i64 0
2729 ; CHECK-NEXT: [[IDXS1:%.*]] = getelementptr inbounds i64, i64* [[S]], i64 1
2830 ; CHECK-NEXT: [[IDXS2:%.*]] = getelementptr inbounds i64, i64* [[S]], i64 2
2931 ; CHECK-NEXT: [[IDXS3:%.*]] = getelementptr inbounds i64, i64* [[S]], i64 3
30 ; CHECK-NEXT: [[TMP8:%.*]] = bitcast i64* [[IDXS0]] to <4 x i64>*
31 ; CHECK-NEXT: store <4 x i64> [[TMP7]], <4 x i64>* [[TMP8]], align 8
32 ; CHECK-NEXT: [[TMP9:%.*]] = bitcast i64* [[IDXS0]] to <4 x i64>*
33 ; CHECK-NEXT: store <4 x i64> [[TMP8]], <4 x i64>* [[TMP9]], align 8
3234 ; CHECK-NEXT: ret void
3335 ;
3436 entry:
6062 ; S[2] = %v5 + %v1
6163 ; S[3] = %v1 + %v4
6264 ;
63 ; TODO: We should broadcast %v1.
65 ; We broadcast %v1.
66
6467 ;
6568 define void @bcast_vals2(i16 *%A, i16 *%B, i16 *%C, i16 *%D, i16 *%E, i32 *%S) {
6669 ; CHECK-LABEL: @bcast_vals2(
7174 ; CHECK-NEXT: [[D0:%.*]] = load i16, i16* [[D:%.*]], align 8
7275 ; CHECK-NEXT: [[E0:%.*]] = load i16, i16* [[E:%.*]], align 8
7376 ; CHECK-NEXT: [[V1:%.*]] = sext i16 [[A0]] to i32
74 ; CHECK-NEXT: [[V2:%.*]] = sext i16 [[B0]] to i32
75 ; CHECK-NEXT: [[V3:%.*]] = sext i16 [[C0]] to i32
76 ; CHECK-NEXT: [[V4:%.*]] = sext i16 [[D0]] to i32
77 ; CHECK-NEXT: [[V5:%.*]] = sext i16 [[E0]] to i32
78 ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[V1]], i32 0
79 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[V3]], i32 1
80 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[V5]], i32 2
81 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[V1]], i32 3
82 ; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> undef, i32 [[V2]], i32 0
83 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[V1]], i32 1
84 ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V1]], i32 2
85 ; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[V4]], i32 3
86 ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP3]], [[TMP7]]
77 ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i16> undef, i16 [[B0]], i32 0
78 ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> [[TMP0]], i16 [[C0]], i32 1
79 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[E0]], i32 2
80 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> [[TMP2]], i16 [[D0]], i32 3
81 ; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[TMP3]] to <4 x i32>
82 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> undef, i32 [[V1]], i32 0
83 ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V1]], i32 1
84 ; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[V1]], i32 2
85 ; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[V1]], i32 3
86 ; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP8]], [[TMP4]]
8787 ; CHECK-NEXT: [[IDXS0:%.*]] = getelementptr inbounds i32, i32* [[S:%.*]], i64 0
8888 ; CHECK-NEXT: [[IDXS1:%.*]] = getelementptr inbounds i32, i32* [[S]], i64 1
8989 ; CHECK-NEXT: [[IDXS2:%.*]] = getelementptr inbounds i32, i32* [[S]], i64 2
9090 ; CHECK-NEXT: [[IDXS3:%.*]] = getelementptr inbounds i32, i32* [[S]], i64 3
91 ; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32* [[IDXS0]] to <4 x i32>*
92 ; CHECK-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP9]], align 8
91 ; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32* [[IDXS0]] to <4 x i32>*
92 ; CHECK-NEXT: store <4 x i32> [[TMP9]], <4 x i32>* [[TMP10]], align 8
9393 ; CHECK-NEXT: ret void
9494 ;
9595 entry: