llvm.org GIT mirror llvm / dad922a
Use profile summary to disable peeling for huge working sets Summary: Detect when the working set size of a profiled application is huge, by comparing the number of counts required to reach the hot percentile in the profile summary to a large threshold*. When the working set size is determined to be huge, disable peeling to avoid bloating the working set further. *Note that the selected threshold (15K) is significantly larger than the largest working set value in SPEC cpu2006 (which is gcc at around 11K). Reviewers: davidxl Subscribers: mehdi_amini, mzolotukhin, eraman, llvm-commits Differential Revision: https://reviews.llvm.org/D36288 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@310005 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson 2 years ago
6 changed file(s) with 87 addition(s) and 32 deletion(s). Raw diff Collapse all Expand all
4848 void computeThresholds();
4949 // Count thresholds to answer isHotCount and isColdCount queries.
5050 Optional HotCountThreshold, ColdCountThreshold;
51 // True if the working set size of the code is considered huge,
52 // because the number of profile counts required to reach the hot
53 // percentile is above a huge threshold.
54 Optional HasHugeWorkingSetSize;
5155
5256 public:
5357 ProfileSummaryInfo(Module &M) : M(M) {}
8387 /// Returns the profile count for \p CallInst.
8488 Optional getProfileCount(const Instruction *CallInst,
8589 BlockFrequencyInfo *BFI);
90 /// Returns true if the working set size of the code is considered huge.
91 bool hasHugeWorkingSetSize();
8692 /// \brief Returns true if \p F has hot function entry.
8793 bool isFunctionEntryHot(const Function *F);
8894 /// Returns true if \p F has hot function entry or hot call edge.
4343 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
4444 "callsite as cold. Otherwise, treat un-sampled callsites as if "
4545 "we have no profile."));
46
47 // Find the minimum count to reach a desired percentile of counts.
48 static uint64_t getMinCountForPercentile(SummaryEntryVector &DS,
49 uint64_t Percentile) {
46 static cl::opt ProfileSummaryHugeWorkingSetSizeThreshold(
47 "profile-summary-huge-working-set-size-threshold", cl::Hidden,
48 cl::init(15000), cl::ZeroOrMore,
49 cl::desc("The code working set size is considered huge if the number of"
50 " blocks required to reach the -profile-summary-cutoff-hot"
51 " percentile exceeds this count."));
52
53 // Find the summary entry for a desired percentile of counts.
54 static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS,
55 uint64_t Percentile) {
5056 auto Compare = [](const ProfileSummaryEntry &Entry, uint64_t Percentile) {
5157 return Entry.Cutoff < Percentile;
5258 };
5561 // detailed summary.
5662 if (It == DS.end())
5763 report_fatal_error("Desired percentile exceeds the maximum cutoff");
58 return It->MinCount;
64 return *It;
5965 }
6066
6167 // The profile summary metadata may be attached either by the frontend or by
168174 if (!computeSummary())
169175 return;
170176 auto &DetailedSummary = Summary->getDetailedSummary();
171 HotCountThreshold =
172 getMinCountForPercentile(DetailedSummary, ProfileSummaryCutoffHot);
173 ColdCountThreshold =
174 getMinCountForPercentile(DetailedSummary, ProfileSummaryCutoffCold);
177 auto &HotEntry =
178 getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot);
179 HotCountThreshold = HotEntry.MinCount;
180 auto &ColdEntry =
181 getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold);
182 ColdCountThreshold = ColdEntry.MinCount;
183 HasHugeWorkingSetSize =
184 HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
185 }
186
187 bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
188 if (!HasHugeWorkingSetSize)
189 computeThresholds();
190 return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue();
175191 }
176192
177193 bool ProfileSummaryInfo::isHotCount(uint64_t C) {
2020 #include "llvm/Analysis/LoopPass.h"
2121 #include "llvm/Analysis/LoopUnrollAnalyzer.h"
2222 #include "llvm/Analysis/OptimizationDiagnosticInfo.h"
23 #include "llvm/Analysis/ProfileSummaryInfo.h"
2324 #include "llvm/Analysis/ScalarEvolution.h"
2425 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
2526 #include "llvm/IR/DataLayout.h"
12511252 auto &AC = AM.getResult(F);
12521253 auto &ORE = AM.getResult(F);
12531254
1255 const ModuleAnalysisManager &MAM =
1256 AM.getResult(F).getManager();
1257 ProfileSummaryInfo *PSI =
1258 MAM.getCachedResult(*F.getParent());
1259
12541260 bool Changed = false;
12551261
12561262 // The unroller requires loops to be in simplified form, and also needs LCSSA.
12791285 // states we support: partial and full (or "simple") unrolling. However, to
12801286 // enable these things we actually pass "None" in for the optional to avoid
12811287 // providing an explicit choice.
1282 Optional AllowPartialParam, RuntimeParam, UpperBoundParam;
1283 bool CurChanged = tryToUnrollLoop(
1284 &L, DT, &LI, SE, TTI, AC, ORE,
1285 /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
1286 /*Threshold*/ None, AllowPartialParam, RuntimeParam, UpperBoundParam,
1287 /*AllowPeeling*/ None);
1288 Optional AllowPartialParam, RuntimeParam, UpperBoundParam,
1289 AllowPeeling;
1290 // Check if the profile summary indicates that the profiled application
1291 // has a huge working set size, in which case we disable peeling to avoid
1292 // bloating it further.
1293 if (PSI && PSI->hasHugeWorkingSetSize())
1294 AllowPeeling = false;
1295 bool CurChanged =
1296 tryToUnrollLoop(&L, DT, &LI, SE, TTI, AC, ORE,
1297 /*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
1298 /*Threshold*/ None, AllowPartialParam, RuntimeParam,
1299 UpperBoundParam, AllowPeeling);
12881300 Changed |= CurChanged;
12891301
12901302 // The parent must not be damaged by unrolling!
197197 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
198198 ; CHECK-O-NEXT: Running pass: InstCombinePass
199199 ; CHECK-O-NEXT: Running pass: LoopUnrollPass
200 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
200201 ; CHECK-O-NEXT: Running pass: InstCombinePass
201202 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
202203 ; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
184184 ; CHECK-POSTLINK-O-NEXT: Running pass: SimplifyCFGPass
185185 ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
186186 ; CHECK-POSTLINK-O-NEXT: Running pass: LoopUnrollPass
187 ; CHECK-POSTLINK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
187188 ; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
188189 ; CHECK-POSTLINK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}OptimizationRemarkEmitterAnalysis
189190 ; CHECK-POSTLINK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LICMPass
0 ; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s
1 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<opt-remark-emit>,unroll' 2>&1 | FileCheck %s
1 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<profile-summary>,function(require,unroll)' 2>&1 | FileCheck %s
2 ; Confirm that peeling is disabled if the number of counts required to reach
3 ; the hot percentile is above the threshold.
4 ; RUN: opt < %s -S -profile-summary-huge-working-set-size-threshold=9 -debug-only=loop-unroll -passes='require,function(require,unroll)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
25 ; REQUIRES: asserts
36
47 ; Make sure we use the profile information correctly to peel-off 3 iterations
1013 ; CHECK-NOT: PEELING
1114
1215 ; Confirm that no peeling occurs when we are performing full unrolling.
13 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=FULLUNROLL
14 ; FULLUNROLL-NOT: PEELING
16 ; RUN: opt < %s -S -debug-only=loop-unroll -passes='require,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
17 ; NOPEEL-NOT: PEELING
1518
1619 ; CHECK-LABEL: @basic
17 ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !1
20 ; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !15
1821 ; CHECK: [[NEXT0]]:
19 ; CHECK: br i1 %{{.*}}, label %[[NEXT1:.*]], label %for.cond.for.end_crit_edge, !prof !2
22 ; CHECK: br i1 %{{.*}}, label %[[NEXT1:.*]], label %for.cond.for.end_crit_edge, !prof !16
2023 ; CHECK: [[NEXT1]]:
21 ; CHECK: br i1 %{{.*}}, label %[[NEXT2:.*]], label %for.cond.for.end_crit_edge, !prof !3
24 ; CHECK: br i1 %{{.*}}, label %[[NEXT2:.*]], label %for.cond.for.end_crit_edge, !prof !17
2225 ; CHECK: [[NEXT2]]:
23 ; CHECK: br i1 %{{.*}}, label %for.body, label %{{.*}}, !prof !4
26 ; CHECK: br i1 %{{.*}}, label %for.body, label %{{.*}}, !prof !18
2427
25 define void @basic(i32* %p, i32 %k) #0 !prof !0 {
28 define void @basic(i32* %p, i32 %k) #0 !prof !15 {
2629 entry:
2730 %cmp3 = icmp slt i32 0, %k
2831 br i1 %cmp3, label %for.body.lr.ph, label %for.end
3740 store i32 %i.05, i32* %p.addr.04, align 4
3841 %inc = add nsw i32 %i.05, 1
3942 %cmp = icmp slt i32 %inc, %k
40 br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !1
43 br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !16
4144
4245 for.cond.for.end_crit_edge: ; preds = %for.body
4346 br label %for.end
5356 ; CHECK: for.body:
5457 ; CHECK-NOT: br
5558 ; CHECK: br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
56 define void @optsize(i32* %p, i32 %k) #1 !prof !0 {
59 define void @optsize(i32* %p, i32 %k) #1 !prof !15 {
5760 entry:
5861 %cmp3 = icmp slt i32 0, %k
5962 br i1 %cmp3, label %for.body.lr.ph, label %for.end
6871 store i32 %i.05, i32* %p.addr.04, align 4
6972 %inc = add nsw i32 %i.05, 1
7073 %cmp = icmp slt i32 %inc, %k
71 br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !1
74 br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !16
7275
7376 for.cond.for.end_crit_edge: ; preds = %for.body
7477 br label %for.end
8083 attributes #0 = { nounwind }
8184 attributes #1 = { nounwind optsize }
8285
83 !0 = !{!"function_entry_count", i64 1}
84 !1 = !{!"branch_weights", i32 3001, i32 1001}
86 !llvm.module.flags = !{!1}
8587
86 ;CHECK: !1 = !{!"branch_weights", i32 900, i32 101}
87 ;CHECK: !2 = !{!"branch_weights", i32 540, i32 360}
88 ;CHECK: !3 = !{!"branch_weights", i32 162, i32 378}
89 ;CHECK: !4 = !{!"branch_weights", i32 1399, i32 162}
88 !1 = !{i32 1, !"ProfileSummary", !2}
89 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
90 !3 = !{!"ProfileFormat", !"InstrProf"}
91 !4 = !{!"TotalCount", i64 10}
92 !5 = !{!"MaxCount", i64 3}
93 !6 = !{!"MaxInternalCount", i64 1}
94 !7 = !{!"MaxFunctionCount", i64 3}
95 !8 = !{!"NumCounts", i64 2}
96 !9 = !{!"NumFunctions", i64 2}
97 !10 = !{!"DetailedSummary", !11}
98 !11 = !{!12, !13, !14}
99 !12 = !{i32 10000, i64 3, i32 2}
100 !13 = !{i32 999000, i64 1, i32 10}
101 !14 = !{i32 999999, i64 1, i32 10}
102 !15 = !{!"function_entry_count", i64 1}
103 !16 = !{!"branch_weights", i32 3001, i32 1001}
90104
105 ;CHECK: !15 = !{!"branch_weights", i32 900, i32 101}
106 ;CHECK: !16 = !{!"branch_weights", i32 540, i32 360}
107 ;CHECK: !17 = !{!"branch_weights", i32 162, i32 378}
108 ;CHECK: !18 = !{!"branch_weights", i32 1399, i32 162}
109