llvm.org GIT mirror llvm / 584a08b
[PGO] Change hardcoded thresholds for cold/inlinehint to use summary Summary: The PGO counter reading will add cold and inlinehint (hot) attributes to functions that are very cold or hot. This was using hardcoded thresholds, instead of the profile summary cutoffs which are used in other hot/cold detection and are more dynamic and adaptable. Switch to using the summary-based cold/hot detection. The hardcoded limits were causing some code that had a medium level of hotness (per the summary) to be incorrectly marked with a cold attribute, blocking inlining. Reviewers: davidxl Subscribers: llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D67673 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@372189 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson a month ago
4 changed file(s) with 60 addition(s) and 34 deletion(s). Raw diff Collapse all Expand all
782782 SummaryData->get(Summary::TotalNumFunctions));
783783 return Cur + SummarySize;
784784 } else {
785 // For older version of profile data, we need to compute on the fly:
786 using namespace IndexedInstrProf;
787
785 // The older versions do not support a profile summary. This just computes
786 // an empty summary, which will not result in accurate hot/cold detection.
787 // We would need to call addRecord for all NamedInstrProfRecords to get the
788 // correct summary. However, this version is old (prior to early 2016) and
789 // has not been supporting an accurate summary for several years.
788790 InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
789 // FIXME: This only computes an empty summary. Need to call addRecord for
790 // all NamedInstrProfRecords to get the correct summary.
791 this->Summary = Builder.getSummary();
791 Summary = Builder.getSummary();
792792 return Cur;
793793 }
794794 }
984984 public:
985985 PGOUseFunc(Function &Func, Module *Modu,
986986 std::unordered_multimap &ComdatMembers,
987 BranchProbabilityInfo *BPI = nullptr,
988 BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false)
989 : F(Func), M(Modu), BFI(BFIin),
987 BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
988 ProfileSummaryInfo *PSI, bool IsCS)
989 : F(Func), M(Modu), BFI(BFIin), PSI(PSI),
990990 FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS),
991991 FreqAttr(FFA_Normal), IsCS(IsCS) {}
992992
10411041 Function &F;
10421042 Module *M;
10431043 BlockFrequencyInfo *BFI;
1044 ProfileSummaryInfo *PSI;
10441045
10451046 // This member stores the shared information with class PGOGenFunc.
10461047 FuncPGOInstrumentation FuncInfo;
10781079 // FIXME: This function should be removed once the functionality in
10791080 // the inliner is implemented.
10801081 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
1081 if (ProgramMaxCount == 0)
1082 return;
1083 // Threshold of the hot functions.
1084 const BranchProbability HotFunctionThreshold(1, 100);
1085 // Threshold of the cold functions.
1086 const BranchProbability ColdFunctionThreshold(2, 10000);
1087 if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
1082 if (PSI->isHotCount(EntryCount))
10881083 FreqAttr = FFA_Hot;
1089 else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
1084 else if (PSI->isColdCount(MaxCount))
10901085 FreqAttr = FFA_Cold;
10911086 }
10921087 };
15951590 static bool annotateAllFunctions(
15961591 Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
15971592 function_ref LookupBPI,
1598 function_ref LookupBFI, bool IsCS) {
1593 function_ref LookupBFI,
1594 ProfileSummaryInfo *PSI, bool IsCS) {
15991595 LLVM_DEBUG(dbgs() << "Read in profile counters: ");
16001596 auto &Ctx = M.getContext();
16011597 // Read the counter array from file.
16251621 ProfileFileName.data(), "Not an IR level instrumentation profile"));
16261622 return false;
16271623 }
1624
1625 // Add the profile summary (read from the header of the indexed summary) here
1626 // so that we can use it below when reading counters (which checks if the
1627 // function should be marked with a cold or inlinehint attribute).
1628 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
1629 IsCS ? ProfileSummary::PSK_CSInstr
1630 : ProfileSummary::PSK_Instr);
16281631
16291632 std::unordered_multimap ComdatMembers;
16301633 collectComdatMembers(M, ComdatMembers);
16381641 // Split indirectbr critical edges here before computing the MST rather than
16391642 // later in getInstrBB() to avoid invalidating it.
16401643 SplitIndirectBrCriticalEdges(F, BPI, BFI);
1641 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS);
1644 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, PSI, IsCS);
16421645 bool AllZeros = false;
16431646 if (!Func.readCounters(PGOReader.get(), AllZeros))
16441647 continue;
16861689 }
16871690 }
16881691 }
1689 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
1690 IsCS ? ProfileSummary::PSK_CSInstr
1691 : ProfileSummary::PSK_Instr);
16921692
16931693 // Set function hotness attribute from the profile.
16941694 // We have to apply these attributes at the end because their presence
17301730 return &FAM.getResult(F);
17311731 };
17321732
1733 auto *PSI = &AM.getResult(M);
1734
17331735 if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
1734 LookupBPI, LookupBFI, IsCS))
1736 LookupBPI, LookupBFI, PSI, IsCS))
17351737 return PreservedAnalyses::all();
17361738
17371739 return PreservedAnalyses::none();
17481750 return &this->getAnalysis(F).getBFI();
17491751 };
17501752
1751 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI,
1753 auto *PSI = &getAnalysis().getPSI();
1754 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, PSI,
17521755 IsCS);
17531756 }
17541757
0 # IR level Instrumentation Flag
11 :ir
2 foo
2 hot
33 # Func Hash:
44 12884901887
55 # Num Counters:
66 1
77 # Counter Values:
8 9999
8 9000
99
10 bar
10 cold
1111 # Func Hash:
1212 12884901887
1313 # Num Counters:
1414 1
1515 # Counter Values:
16 0
16 10
17
18 med
19 # Func Hash:
20 12884901887
21 # Num Counters:
22 1
23 # Counter Values:
24 50
55
66 @s = common dso_local local_unnamed_addr global i32 0, align 4
77
8 define void @bar() {
9 ; CHECK-LABEL: @bar
8 define void @cold() {
9 ; CHECK-LABEL: @cold()
10 ; CHECK-SAME: #[[COLD_ATTR:[0-1]+]]
1011 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_ZERO:[0-9]+]]
1112
1213 entry:
1415 ret void
1516 }
1617
17 define void @foo() {
18 ; CHECK-LABEL: @foo
18 define void @hot() {
19 ; CHECK-LABEL: @hot()
20 ; CHECK-SAME: #[[HOT_ATTR:[0-1]+]]
1921 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_NON_ZERO:[0-9]+]]
2022 entry:
2123 %0 = load i32, i32* @s, align 4
2426 ret void
2527 }
2628
27 ; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 0}
28 ; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9999}
29 define void @med() {
30 ; CHECK-LABEL: @med
31 ; CHECK-NOT: #
32 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_MED:[0-9]+]]
33
34 entry:
35 store i32 1, i32* @s, align 4
36 ret void
37 }
38
39 ; CHECK-DAG: attributes #[[COLD_ATTR]] = { cold }
40 ; CHECK-DAG: attributes #[[HOT_ATTR]] = { inlinehint }
41 ; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 10}
42 ; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9000}
43 ; CHECK-DAG: ![[FUNC_ENTRY_COUNT_MED]] = !{!"function_entry_count", i64 50}