llvm.org GIT mirror llvm / 03a6f4a
[PGO] add MST min edge selection heuristic to ensure non-zero entry count Differential Revision: http://reviews.llvm.org/D41059 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@320998 91177308-0d34-0410-b5e6-96231b3b80d8 Xinliang David Li 1 year, 9 months ago
10 changed file(s) with 102 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
4545 // This map records the auxiliary information for each BB.
4646 DenseMap> BBInfos;
4747
48 // Whehter the function has an exit block with no successors.
49 // (For function with an infinite loop, this block may be absent)
50 bool ExitBlockFound = false;
51
4852 // Find the root group of the G and compress the path from G to the root.
4953 BBInfo *findAndCompressGroup(BBInfo *G) {
5054 if (G->Group != G)
9498 void buildEdges() {
9599 DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n");
96100
97 const BasicBlock *BB = &(F.getEntryBlock());
101 const BasicBlock *Entry = &(F.getEntryBlock());
98102 uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
103 Edge *EntryIncoming = nullptr, *EntryOutgoing = nullptr,
104 *ExitOutgoing = nullptr, *ExitIncoming = nullptr;
105 uint64_t MaxEntryOutWeight = 0, MaxExitOutWeight = 0, MaxExitInWeight = 0;
106
99107 // Add a fake edge to the entry.
100 addEdge(nullptr, BB, EntryWeight);
108 EntryIncoming = &addEdge(nullptr, Entry, EntryWeight);
109 DEBUG(dbgs() << " Edge: from fake node to " << Entry->getName()
110 << " w = " << EntryWeight << "\n");
101111
102112 // Special handling for single BB functions.
103 if (succ_empty(BB)) {
104 addEdge(BB, nullptr, EntryWeight);
113 if (succ_empty(Entry)) {
114 addEdge(Entry, nullptr, EntryWeight);
105115 return;
106116 }
107117
125135 }
126136 if (BPI != nullptr)
127137 Weight = BPI->getEdgeProbability(&*BB, TargetBB).scale(scaleFactor);
128 addEdge(&*BB, TargetBB, Weight).IsCritical = Critical;
138 auto *E = &addEdge(&*BB, TargetBB, Weight);
139 E->IsCritical = Critical;
129140 DEBUG(dbgs() << " Edge: from " << BB->getName() << " to "
130141 << TargetBB->getName() << " w=" << Weight << "\n");
142
143 // Keep track of entry/exit edges:
144 if (&*BB == Entry) {
145 if (Weight > MaxEntryOutWeight) {
146 MaxEntryOutWeight = Weight;
147 EntryOutgoing = E;
148 }
149 }
150
151 auto *TargetTI = TargetBB->getTerminator();
152 if (TargetTI && !TargetTI->getNumSuccessors()) {
153 if (Weight > MaxExitInWeight) {
154 MaxExitInWeight = Weight;
155 ExitIncoming = E;
156 }
157 }
131158 }
132159 } else {
133 addEdge(&*BB, nullptr, BBWeight);
134 DEBUG(dbgs() << " Edge: from " << BB->getName() << " to exit"
160 ExitBlockFound = true;
161 Edge *ExitO = &addEdge(&*BB, nullptr, BBWeight);
162 if (BBWeight > MaxExitOutWeight) {
163 MaxExitOutWeight = BBWeight;
164 ExitOutgoing = ExitO;
165 }
166 DEBUG(dbgs() << " Edge: from " << BB->getName() << " to fake exit"
135167 << " w = " << BBWeight << "\n");
136168 }
169 }
170
171 // Entry/exit edge adjustment heurisitic:
172 // prefer instrumenting entry edge over exit edge
173 // if possible. Those exit edges may never have a chance to be
174 // executed (for instance the program is an event handling loop)
175 // before the profile is asynchronously dumped.
176 //
177 // If EntryIncoming and ExitOutgoing has similar weight, make sure
178 // ExitOutging is selected as the min-edge. Similarly, if EntryOutgoing
179 // and ExitIncoming has similar weight, make sure ExitIncoming becomes
180 // the min-edge.
181 uint64_t EntryInWeight = EntryWeight;
182
183 if (EntryInWeight >= MaxExitOutWeight &&
184 EntryInWeight * 2 < MaxExitOutWeight * 3) {
185 EntryIncoming->Weight = MaxExitOutWeight;
186 ExitOutgoing->Weight = EntryInWeight + 1;
187 }
188
189 if (MaxEntryOutWeight >= MaxExitInWeight &&
190 MaxEntryOutWeight * 2 < MaxExitInWeight * 3) {
191 EntryOutgoing->Weight = MaxExitInWeight;
192 ExitIncoming->Weight = MaxEntryOutWeight + 1;
137193 }
138194 }
139195
166222 for (auto &Ei : AllEdges) {
167223 if (Ei->Removed)
168224 continue;
225 // If we detect infinite loops, force
226 // instrumenting the entry edge:
227 if (!ExitBlockFound && Ei->SrcBB == nullptr)
228 continue;
169229 if (unionGroups(Ei->SrcBB, Ei->DestBB))
170230 Ei->InMST = true;
171231 }
55 # Num Counters:
66 3
77 # Counter Values:
8 20
8 21
99 21
1010 0
3131 ; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]]
3232 entry:
3333 ; GEN: entry:
34 ; GEN-NOT: llvm.instrprof.increment
34 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 0)
3535 %cmp = icmp sgt i32 %i, 0
3636 br i1 %cmp, label %if.then, label %if.end
3737 ; USE: br i1 %cmp, label %if.then, label %if.end
4949
5050 if.end:
5151 ; GEN: if.end:
52 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 0)
52 ; GEN-NOT: llvm.instrprof.increment
53 ; GEN: ret i32
5354 %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
5455 ret i32 %retv
5556 }
44
55 define void @foo(i32 %n, i32 %N) {
66 ; PROMO-LABEL: @foo
7 ; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
8 ; PROMO-NEXT: add
9 ; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
710 bb:
811 %tmp = add nsw i32 %n, 1
912 %tmp1 = add nsw i32 %n, -1
5659 ; ATOMIC_PROMO: atomicrmw add {{.*}} @__profc_foo{{.*}}0), i64 %[[LIVEOUT1]] seq_cst
5760 ; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}1), i64 %[[LIVEOUT2]] seq_cst
5861 ; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}2), i64 %[[LIVEOUT3]] seq_cst
59 ; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
60 ; PROMO-NEXT: add
61 ; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
6262 ; PROMO-NOT: @__profc_foo
6363
6464
0 ; RUN: opt < %s -pgo-instr-gen -S -o - | FileCheck %s
1
2 define void @foo() {
3 entry:
4 br label %while.body
5 ; CHECK: llvm.instrprof.increment
6
7 while.body: ; preds = %entry, %while.body
8 ; CHECK: llvm.instrprof.increment
9 call void (...) @bar() #2
10 br label %while.body
11 }
12
13 declare void @bar(...)
14
15 attributes #0 = { nounwind }
16
1515 define i32 @bar(i32 %i) {
1616 entry:
1717 ; GEN: entry:
18 ; GEN-NOT: call void @llvm.instrprof.increment
18 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 0)
1919 %rem = srem i32 %i, 3
2020 %tobool = icmp ne i32 %rem, 0
2121 br i1 %tobool, label %if.then, label %if.end
3333
3434 if.end:
3535 ; GEN: if.end:
36 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 0)
36 ; GEN-NOT: call void @llvm.instrprof.increment
37 ; GEN: ret i32
3738 ret i32 0
3839 }
3940
1212 define i32 @test_simple_for(i32 %n) {
1313 entry:
1414 ; GEN: entry:
15 ; GEN-NOT: call void @llvm.instrprof.increment
15 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
1616 br label %for.cond
1717
1818 for.cond:
4040
4141 for.end:
4242 ; GEN: for.end:
43 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
43 ; GEN-NOT: call void @llvm.instrprof.increment
44 ; GEN: ret i32
4445 ret i32 %sum
4546 }
1212 define i32 @test_nested_for(i32 %r, i32 %s) {
1313 entry:
1414 ; GEN: entry:
15 ; GEN-NOT: call void @llvm.instrprof.increment
15 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
1616 br label %for.cond.outer
1717
1818 for.cond.outer:
6464
6565 for.end.outer:
6666 ; GEN: for.end.outer:
67 ; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
67 ; GEN-NOT: call void @llvm.instrprof.increment
68 ; GEN: ret i32
6869 ret i32 %sum.0
6970 }
7071
4141 ret i32 %mul
4242 }
4343 ; USE: ![[BW_ENTRY]] = !{!"branch_weights", i32 21, i32 0}
44 ; USE: ![[BW_IF]] = !{!"branch_weights", i32 0, i32 20}
44 ; USE: ![[BW_IF]] = !{!"branch_weights", i32 0, i32 21}