llvm.org GIT mirror llvm / 331d5b9
[SimpleLoopUnswitch] Implement handling of prof branch_weights metadata for SwitchInst Differential Revision: https://reviews.llvm.org/D60606 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@364734 91177308-0d34-0410-b5e6-96231b3b80d8 Yevgeny Rouban 2 months ago
3 changed file(s) with 302 addition(s) and 18 deletion(s). Raw diff Collapse all Expand all
593593 ExitCaseIndices.push_back(Case.getCaseIndex());
594594 }
595595 BasicBlock *DefaultExitBB = nullptr;
596 SwitchInstProfUpdateWrapper::CaseWeightOpt DefaultCaseWeight =
597 SwitchInstProfUpdateWrapper::getSuccessorWeight(SI, 0);
596598 if (!L.contains(SI.getDefaultDest()) &&
597599 areLoopExitPHIsLoopInvariant(L, *ParentBB, *SI.getDefaultDest()) &&
598 !isa(SI.getDefaultDest()->getTerminator()))
600 !isa(SI.getDefaultDest()->getTerminator())) {
599601 DefaultExitBB = SI.getDefaultDest();
600 else if (ExitCaseIndices.empty())
602 } else if (ExitCaseIndices.empty())
601603 return false;
602604
603605 LLVM_DEBUG(dbgs() << " unswitching trivial switch...\n");
621623
622624 // Store the exit cases into a separate data structure and remove them from
623625 // the switch.
624 SmallVectorpair, 4> ExitCases;
626 SmallVectortuple
627 SwitchInstProfUpdateWrapper::CaseWeightOpt>,
628 4> ExitCases;
625629 ExitCases.reserve(ExitCaseIndices.size());
630 SwitchInstProfUpdateWrapper SIW(SI);
626631 // We walk the case indices backwards so that we remove the last case first
627632 // and don't disrupt the earlier indices.
628633 for (unsigned Index : reverse(ExitCaseIndices)) {
632637 if (!ExitL || ExitL->contains(OuterL))
633638 OuterL = ExitL;
634639 // Save the value of this case.
635 ExitCases.push_back({CaseI->getCaseValue(), CaseI->getCaseSuccessor()});
640 auto W = SIW.getSuccessorWeight(CaseI->getSuccessorIndex());
641 ExitCases.emplace_back(CaseI->getCaseValue(), CaseI->getCaseSuccessor(), W);
636642 // Delete the unswitched cases.
637 SI.removeCase(CaseI);
643 SIW.removeCase(CaseI);
638644 }
639645
640646 if (SE) {
672678
673679 // Now add the unswitched switch.
674680 auto *NewSI = SwitchInst::Create(LoopCond, NewPH, ExitCases.size(), OldPH);
681 SwitchInstProfUpdateWrapper NewSIW(*NewSI);
675682
676683 // Rewrite the IR for the unswitched basic blocks. This requires two steps.
677684 // First, we split any exit blocks with remaining in-loop predecessors. Then
699706 }
700707 // Note that we must use a reference in the for loop so that we update the
701708 // container.
702 for (auto &CasePair : reverse(ExitCases)) {
709 for (auto &ExitCase : reverse(ExitCases)) {
703710 // Grab a reference to the exit block in the pair so that we can update it.
704 BasicBlock *ExitBB = CasePair.second;
711 BasicBlock *ExitBB = std::get<1>(ExitCase);
705712
706713 // If this case is the last edge into the exit block, we can simply reuse it
707714 // as it will no longer be a loop exit. No mapping necessary.
723730 /*FullUnswitch*/ true);
724731 }
725732 // Update the case pair to point to the split block.
726 CasePair.second = SplitExitBB;
733 std::get<1>(ExitCase) = SplitExitBB;
727734 }
728735
729736 // Now add the unswitched cases. We do this in reverse order as we built them
730737 // in reverse order.
731 for (auto CasePair : reverse(ExitCases)) {
732 ConstantInt *CaseVal = CasePair.first;
733 BasicBlock *UnswitchedBB = CasePair.second;
734
735 NewSI->addCase(CaseVal, UnswitchedBB);
738 for (auto &ExitCase : reverse(ExitCases)) {
739 ConstantInt *CaseVal = std::get<0>(ExitCase);
740 BasicBlock *UnswitchedBB = std::get<1>(ExitCase);
741
742 NewSIW.addCase(CaseVal, UnswitchedBB, std::get<2>(ExitCase));
736743 }
737744
738745 // If the default was unswitched, re-point it and add explicit cases for
739746 // entering the loop.
740747 if (DefaultExitBB) {
741 NewSI->setDefaultDest(DefaultExitBB);
748 NewSIW->setDefaultDest(DefaultExitBB);
749 NewSIW.setSuccessorWeight(0, DefaultCaseWeight);
742750
743751 // We removed all the exit cases, so we just copy the cases to the
744752 // unswitched switch.
745 for (auto Case : SI.cases())
746 NewSI->addCase(Case.getCaseValue(), NewPH);
753 for (const auto &Case : SI.cases())
754 NewSIW.addCase(Case.getCaseValue(), NewPH,
755 SIW.getSuccessorWeight(Case.getSuccessorIndex()));
756 } else if (DefaultCaseWeight) {
757 // We have to set branch weight of the default case.
758 uint64_t SW = *DefaultCaseWeight;
759 for (const auto &Case : SI.cases()) {
760 auto W = SIW.getSuccessorWeight(Case.getSuccessorIndex());
761 assert(W &&
762 "case weight must be defined as default case weight is defined");
763 SW += *W;
764 }
765 NewSIW.setSuccessorWeight(0, SW);
747766 }
748767
749768 // If we ended up with a common successor for every path through the switch
768787 /*KeepOneInputPHIs*/ true);
769788 }
770789 // Now nuke the switch and replace it with a direct branch.
771 SI.eraseFromParent();
790 SIW.eraseFromParent();
772791 BranchInst::Create(CommonSuccBB, BB);
773792 } else if (DefaultExitBB) {
774793 assert(SI.getNumCases() > 0 &&
778797 // being simple and keeping the number of edges from this switch to
779798 // successors the same, and avoiding any PHI update complexity.
780799 auto LastCaseI = std::prev(SI.case_end());
800
781801 SI.setDefaultDest(LastCaseI->getCaseSuccessor());
782 SI.removeCase(LastCaseI);
802 SIW.setSuccessorWeight(
803 0, SIW.getSuccessorWeight(LastCaseI->getSuccessorIndex()));
804 SIW.removeCase(LastCaseI);
783805 }
784806
785807 // Walk the unswitched exit blocks and the unswitched split blocks and update
0 ; RUN: opt -passes='loop(unswitch),verify' -S < %s | FileCheck %s
1 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify' -S < %s | FileCheck %s
2
3 declare void @incf()
4 declare void @decf()
5
6 define i32 @test2(i32 %c) {
7 ; CHECK-LABEL: @test2(
8 br label %loop_begin
9
10 ; CHECK: !prof ![[MD0:[0-9]+]]
11 ; CHECK: loop_begin:
12 ; CHECK: !prof ![[MD1:[0-9]+]]
13 loop_begin:
14
15 switch i32 %c, label %default [
16 i32 1, label %inc
17 i32 2, label %dec
18 ], !prof !{!"branch_weights", i32 99, i32 1, i32 2}
19
20 inc:
21 call void @incf()
22 br label %loop_begin
23
24 dec:
25 call void @decf()
26 br label %loop_begin
27
28 default:
29 ret i32 0
30 }
31
32 ; CHECK: ![[MD0]] = !{!"branch_weights", i32 99, i32 1, i32 2}
33 ; CHECK: ![[MD1]] = !{!"branch_weights", i32 2, i32 1}
0 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
1 ; then metadata checks MDn were added manually.
2 ; RUN: opt -passes='loop(unswitch),verify' -S < %s | FileCheck %s
3 ; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify' -S < %s | FileCheck %s
4
5 declare void @some_func()
6
7 ; Test for a trivially unswitchable switch with non-default case exiting.
8 define i32 @test2(i32* %var, i32 %cond1, i32 %cond2) {
9 ; CHECK-LABEL: @test2(
10 ; CHECK-NEXT: entry:
11 ; CHECK-NEXT: switch i32 [[COND2:%.*]], label [[ENTRY_SPLIT:%.*]] [
12 ; CHECK-NEXT: i32 2, label [[LOOP_EXIT2:%.*]]
13 ; CHECK-NEXT: ], !prof ![[MD0:[0-9]+]]
14 ; CHECK: entry.split:
15 ; CHECK-NEXT: br label [[LOOP_BEGIN:%.*]]
16 ; CHECK: loop_begin:
17 ; CHECK-NEXT: [[VAR_VAL:%.*]] = load i32, i32* [[VAR:%.*]]
18 ; CHECK-NEXT: switch i32 [[COND2]], label [[LOOP2:%.*]] [
19 ; CHECK-NEXT: i32 0, label [[LOOP0:%.*]]
20 ; CHECK-NEXT: i32 1, label [[LOOP1:%.*]]
21 ; CHECK-NEXT: ], !prof ![[MD1:[0-9]+]]
22 ; CHECK: loop0:
23 ; CHECK-NEXT: call void @some_func()
24 ; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
25 ; CHECK: loop1:
26 ; CHECK-NEXT: call void @some_func()
27 ; CHECK-NEXT: br label [[LOOP_LATCH]]
28 ; CHECK: loop2:
29 ; CHECK-NEXT: call void @some_func()
30 ; CHECK-NEXT: br label [[LOOP_LATCH]]
31 ; CHECK: loop_latch:
32 ; CHECK-NEXT: br label [[LOOP_BEGIN]]
33 ; CHECK: loop_exit1:
34 ; CHECK-NEXT: ret i32 0
35 ; CHECK: loop_exit2:
36 ; CHECK-NEXT: ret i32 0
37 ; CHECK: loop_exit3:
38 ; CHECK-NEXT: ret i32 0
39 ;
40 entry:
41 br label %loop_begin
42
43 loop_begin:
44 %var_val = load i32, i32* %var
45 switch i32 %cond2, label %loop2 [
46 i32 0, label %loop0
47 i32 1, label %loop1
48 i32 2, label %loop_exit2
49 ], !prof !{!"branch_weights", i32 99, i32 100, i32 101, i32 102}
50
51 loop0:
52 call void @some_func()
53 br label %loop_latch
54
55 loop1:
56 call void @some_func()
57 br label %loop_latch
58
59 loop2:
60 call void @some_func()
61 br label %loop_latch
62
63 loop_latch:
64 br label %loop_begin
65
66 loop_exit1:
67 ret i32 0
68
69 loop_exit2:
70 ret i32 0
71
72 loop_exit3:
73 ret i32 0
74 }
75
76 ; Test for a trivially unswitchable switch with only the default case exiting.
77 define i32 @test3(i32* %var, i32 %cond1, i32 %cond2) {
78 ; CHECK-LABEL: @test3(
79 ; CHECK-NEXT: entry:
80 ; CHECK-NEXT: switch i32 [[COND2:%.*]], label [[LOOP_EXIT2:%.*]] [
81 ; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT:%.*]]
82 ; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT]]
83 ; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT]]
84 ; CHECK-NEXT: ], !prof ![[MD2:[0-9]+]]
85 ; CHECK: entry.split:
86 ; CHECK-NEXT: br label [[LOOP_BEGIN:%.*]]
87 ; CHECK: loop_begin:
88 ; CHECK-NEXT: [[VAR_VAL:%.*]] = load i32, i32* [[VAR:%.*]]
89 ; CHECK-NEXT: switch i32 [[COND2]], label [[LOOP2:%.*]] [
90 ; CHECK-NEXT: i32 0, label [[LOOP0:%.*]]
91 ; CHECK-NEXT: i32 1, label [[LOOP1:%.*]]
92 ; CHECK-NEXT: ], !prof ![[MD3:[0-9]+]]
93 ; CHECK: loop0:
94 ; CHECK-NEXT: call void @some_func()
95 ; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
96 ; CHECK: loop1:
97 ; CHECK-NEXT: call void @some_func()
98 ; CHECK-NEXT: br label [[LOOP_LATCH]]
99 ; CHECK: loop2:
100 ; CHECK-NEXT: call void @some_func()
101 ; CHECK-NEXT: br label [[LOOP_LATCH]]
102 ; CHECK: loop_latch:
103 ; CHECK-NEXT: br label [[LOOP_BEGIN]]
104 ; CHECK: loop_exit1:
105 ; CHECK-NEXT: ret i32 0
106 ; CHECK: loop_exit2:
107 ; CHECK-NEXT: ret i32 0
108 ; CHECK: loop_exit3:
109 ; CHECK-NEXT: ret i32 0
110 ;
111 entry:
112 br label %loop_begin
113
114 loop_begin:
115 %var_val = load i32, i32* %var
116 switch i32 %cond2, label %loop_exit2 [
117 i32 0, label %loop0
118 i32 1, label %loop1
119 i32 2, label %loop2
120 ], !prof !{!"branch_weights", i32 99, i32 100, i32 101, i32 102}
121
122 loop0:
123 call void @some_func()
124 br label %loop_latch
125
126 loop1:
127 call void @some_func()
128 br label %loop_latch
129
130 loop2:
131 call void @some_func()
132 br label %loop_latch
133
134 loop_latch:
135 br label %loop_begin
136
137 loop_exit1:
138 ret i32 0
139
140 loop_exit2:
141 ret i32 0
142
143 loop_exit3:
144 ret i32 0
145 }
146
147 ; Test for a trivially unswitchable switch with multiple exiting cases and
148 ; multiple looping cases.
149 define i32 @test4(i32* %var, i32 %cond1, i32 %cond2) {
150 ; CHECK-LABEL: @test4(
151 ; CHECK-NEXT: entry:
152 ; CHECK-NEXT: switch i32 [[COND2:%.*]], label [[LOOP_EXIT2:%.*]] [
153 ; CHECK-NEXT: i32 13, label [[LOOP_EXIT1:%.*]]
154 ; CHECK-NEXT: i32 42, label [[LOOP_EXIT3:%.*]]
155 ; CHECK-NEXT: i32 0, label [[ENTRY_SPLIT:%.*]]
156 ; CHECK-NEXT: i32 1, label [[ENTRY_SPLIT]]
157 ; CHECK-NEXT: i32 2, label [[ENTRY_SPLIT]]
158 ; CHECK-NEXT: ], !prof ![[MD4:[0-9]+]]
159 ; CHECK: entry.split:
160 ; CHECK-NEXT: br label [[LOOP_BEGIN:%.*]]
161 ; CHECK: loop_begin:
162 ; CHECK-NEXT: [[VAR_VAL:%.*]] = load i32, i32* [[VAR:%.*]]
163 ; CHECK-NEXT: switch i32 [[COND2]], label [[LOOP2:%.*]] [
164 ; CHECK-NEXT: i32 0, label [[LOOP0:%.*]]
165 ; CHECK-NEXT: i32 1, label [[LOOP1:%.*]]
166 ; CHECK-NEXT: ], !prof ![[MD3:[0-9]+]]
167 ; CHECK: loop0:
168 ; CHECK-NEXT: call void @some_func()
169 ; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]
170 ; CHECK: loop1:
171 ; CHECK-NEXT: call void @some_func()
172 ; CHECK-NEXT: br label [[LOOP_LATCH]]
173 ; CHECK: loop2:
174 ; CHECK-NEXT: call void @some_func()
175 ; CHECK-NEXT: br label [[LOOP_LATCH]]
176 ; CHECK: loop_latch:
177 ; CHECK-NEXT: br label [[LOOP_BEGIN]]
178 ; CHECK: loop_exit1:
179 ; CHECK-NEXT: ret i32 0
180 ; CHECK: loop_exit2:
181 ; CHECK-NEXT: ret i32 0
182 ; CHECK: loop_exit3:
183 ; CHECK-NEXT: ret i32 0
184 ;
185 entry:
186 br label %loop_begin
187
188 loop_begin:
189 %var_val = load i32, i32* %var
190 switch i32 %cond2, label %loop_exit2 [
191 i32 0, label %loop0
192 i32 1, label %loop1
193 i32 13, label %loop_exit1
194 i32 2, label %loop2
195 i32 42, label %loop_exit3
196 ], !prof !{!"branch_weights", i32 99, i32 100, i32 101, i32 113, i32 102, i32 142}
197
198 loop0:
199 call void @some_func()
200 br label %loop_latch
201
202 loop1:
203 call void @some_func()
204 br label %loop_latch
205
206 loop2:
207 call void @some_func()
208 br label %loop_latch
209
210 loop_latch:
211 br label %loop_begin
212
213 loop_exit1:
214 ret i32 0
215
216 loop_exit2:
217 ret i32 0
218
219 loop_exit3:
220 ret i32 0
221 }
222
223 ; CHECK: ![[MD0]] = !{!"branch_weights", i32 300, i32 102}
224 ; CHECK: ![[MD1]] = !{!"branch_weights", i32 99, i32 100, i32 101}
225 ; CHECK: ![[MD2]] = !{!"branch_weights", i32 99, i32 100, i32 101, i32 102}
226 ; CHECK: ![[MD3]] = !{!"branch_weights", i32 102, i32 100, i32 101}
227 ; CHECK: ![[MD4]] = !{!"branch_weights", i32 99, i32 113, i32 142, i32 100, i32 101, i32 102}