llvm.org GIT mirror llvm / cd2a5b6
SamplePGO ThinLTO ICP fix for local functions. Summary: In SamplePGO, if the profile is collected from non-LTO binary, and used to drive ThinLTO, the indirect call promotion may fail because ThinLTO adjusts local function names to avoid conflicts. There are two places of where the mismatch can happen: 1. thin-link prepends SourceFileName to front of FuncName to build the GUID (GlobalValue::getGlobalIdentifier). Unlike instrumentation FDO, SamplePGO does not use the PGOFuncName scheme and therefore the indirect call target profile data contains a hash of the OriginalName. 2. backend compiler promotes some local functions to global and appends .llvm.{$ModuleHash} to the end of the FuncName to derive PromotedFunctionName This patch tries at the best effort to find the GUID from the original local function name (in profile), and use that in ICP promotion, and in SamplePGO matching that happens in the backend after importing/inlining: 1. in thin-link, it builds the map from OriginalName to GUID so that when thin-link reads in indirect call target profile (represented by OriginalName), it knows which GUID to import. 2. in backend compiler, if sample profile reader cannot find a profile match for PromotedFunctionName, it will try to find if there is a match for OriginalFunctionName. 3. in backend compiler, we build symbol table entry for OriginalFunctionName and pointer to the same symbol of PromotedFunctionName, so that ICP can find the correct target to promote. Reviewers: mehdi_amini, tejohnson Reviewed By: tejohnson Subscribers: llvm-commits, Prazek Differential Revision: https://reviews.llvm.org/D30754 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@297757 91177308-0d34-0410-b5e6-96231b3b80d8 Dehao Chen 2 years ago
9 changed file(s) with 158 addition(s) and 6 deletion(s). Raw diff Collapse all Expand all
161161 protected:
162162 /// GlobalValueSummary constructor.
163163 GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector Refs)
164 : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {}
164 : Kind(K), Flags(Flags), OriginalName(0), RefEdgeList(std::move(Refs)) {}
165165
166166 public:
167167 virtual ~GlobalValueSummary() = default;
527527 // FIXME: Add bitcode read/write support for this field.
528528 std::map TypeIdMap;
529529
530 /// Mapping from original ID to GUID. If original ID can map to multiple
531 /// GUIDs, it will be mapped to 0.
532 std::map OidGuidMap;
533
530534 // YAML I/O support.
531535 friend yaml::MappingTraits;
532536
554558 return GlobalValueMap.find(ValueGUID);
555559 }
556560
561 /// Return the GUID for \p OriginalId in the OidGuidMap.
562 GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const {
563 const auto I = OidGuidMap.find(OriginalID);
564 return I == OidGuidMap.end() ? 0 : I->second;
565 }
566
557567 /// Add a global value summary for a value of the given name.
558568 void addGlobalValueSummary(StringRef ValueName,
559569 std::unique_ptr Summary) {
570 addOriginalName(GlobalValue::getGUID(ValueName),
571 Summary->getOriginalName());
560572 GlobalValueMap[GlobalValue::getGUID(ValueName)].push_back(
561573 std::move(Summary));
562574 }
564576 /// Add a global value summary for a value of the given GUID.
565577 void addGlobalValueSummary(GlobalValue::GUID ValueGUID,
566578 std::unique_ptr Summary) {
579 addOriginalName(ValueGUID, Summary->getOriginalName());
567580 GlobalValueMap[ValueGUID].push_back(std::move(Summary));
581 }
582
583 /// Add an original name for the value of the given GUID.
584 void addOriginalName(GlobalValue::GUID ValueGUID,
585 GlobalValue::GUID OrigGUID) {
586 if (OrigGUID == 0 || ValueGUID == OrigGUID)
587 return;
588 if (OidGuidMap.count(OrigGUID) && OidGuidMap[OrigGUID] != ValueGUID)
589 OidGuidMap[OrigGUID] = 0;
590 else
591 OidGuidMap[OrigGUID] = ValueGUID;
568592 }
569593
570594 /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if
282282
283283 /// \brief Return the samples collected for function \p F.
284284 FunctionSamples *getSamplesFor(const Function &F) {
285 return &Profiles[F.getName()];
285 // The function name may have been updated by adding suffix. In sample
286 // profile, the function names are all stripped, so we need to strip
287 // the function name suffix before matching with profile.
288 if (Profiles.count(F.getName().split('.').first))
289 return &Profiles[(F.getName().split('.').first)];
290 return nullptr;
286291 }
287292
288293 /// \brief Return all the profiles.
48454845 // Keep around the last seen summary to be used when we see an optional
48464846 // "OriginalName" attachement.
48474847 GlobalValueSummary *LastSeenSummary = nullptr;
4848 GlobalValue::GUID LastSeenGUID = 0;
48484849 bool Combined = false;
48494850
48504851 // We can expect to see any number of type ID information records before
50135014 PendingTypeTestAssumeConstVCalls.clear();
50145015 PendingTypeCheckedLoadConstVCalls.clear();
50155016 LastSeenSummary = FS.get();
5017 LastSeenGUID = GUID;
50165018 FS->setModulePath(ModuleIdMap[ModuleId]);
50175019 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
50185020 Combined = true;
50395041 AS->setAliasee(AliaseeInModule);
50405042
50415043 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5044 LastSeenGUID = GUID;
50425045 TheIndex.addGlobalValueSummary(GUID, std::move(AS));
50435046 Combined = true;
50445047 break;
50555058 LastSeenSummary = FS.get();
50565059 FS->setModulePath(ModuleIdMap[ModuleId]);
50575060 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5061 LastSeenGUID = GUID;
50585062 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
50595063 Combined = true;
50605064 break;
50655069 if (!LastSeenSummary)
50665070 return error("Name attachment that does not follow a combined record");
50675071 LastSeenSummary->setOriginalName(OriginalName);
5072 TheIndex.addOriginalName(LastSeenGUID, OriginalName);
50685073 // Reset the LastSeenSummary
50695074 LastSeenSummary = nullptr;
5075 LastSeenGUID = 0;
50705076 break;
50715077 }
50725078 case bitc::FS_TYPE_TESTS: {
37073707 for (auto &EI : FS->calls()) {
37083708 // If this GUID doesn't have a value id, it doesn't have a function
37093709 // summary and we don't need to record any calls to it.
3710 if (!hasValueId(EI.first.getGUID()))
3711 continue;
3712 NameVals.push_back(getValueId(EI.first.getGUID()));
3710 GlobalValue::GUID GUID = EI.first.getGUID();
3711 if (!hasValueId(GUID)) {
3712 // For SamplePGO, the indirect call targets for local functions will
3713 // have its original name annotated in profile. We try to find the
3714 // corresponding PGOFuncName as the GUID.
3715 GUID = Index.getGUIDFromOriginalID(GUID);
3716 if (GUID == 0 || !hasValueId(GUID))
3717 continue;
3718 }
3719 NameVals.push_back(getValueId(GUID));
37133720 if (HasProfileData)
37143721 NameVals.push_back(static_cast(EI.second.Hotness));
37153722 }
297297 const std::string &PGOFuncName = getPGOFuncName(F, InLTO);
298298 addFuncName(PGOFuncName);
299299 MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F);
300 // In ThinLTO, local function may have been promoted to global and have
301 // suffix added to the function name. We need to add the stripped function
302 // name to the symbol table so that we can find a match from profile.
303 if (InLTO) {
304 auto pos = PGOFuncName.find('.');
305 if (pos != std::string::npos) {
306 const std::string &OtherFuncName = PGOFuncName.substr(0, pos);
307 addFuncName(OtherFuncName);
308 MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F);
309 }
310 }
300311 }
301312
302313 finalizeSymtab();
195195 for (auto &Edge : Summary.calls()) {
196196 auto GUID = Edge.first.getGUID();
197197 DEBUG(dbgs() << " edge -> " << GUID << " Threshold:" << Threshold << "\n");
198
199 if (Index.findGlobalValueSummaryList(GUID) == Index.end()) {
200 // For SamplePGO, the indirect call targets for local functions will
201 // have its original name annotated in profile. We try to find the
202 // corresponding PGOFuncName as the GUID.
203 GUID = Index.getGUIDFromOriginalID(GUID);
204 if (GUID == 0)
205 continue;
206 }
198207
199208 if (DefinedGVSummaries.count(GUID)) {
200209 DEBUG(dbgs() << "ignored! Target already in destination module.\n");
13821382 bool SampleProfileLoader::runOnFunction(Function &F) {
13831383 F.setEntryCount(0);
13841384 Samples = Reader->getSamplesFor(F);
1385 if (!Samples->empty())
1385 if (Samples && !Samples->empty())
13861386 return emitAnnotations(F);
13871387 return false;
13881388 }
0 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
1 target triple = "x86_64-unknown-linux-gnu"
2
3 @fptr = external local_unnamed_addr global void ()*, align 8
4
5 ; Function Attrs: norecurse nounwind uwtable
6 define void @_Z6updatei(i32 %i) local_unnamed_addr #0 {
7 entry:
8 store void ()* @_ZL3foov, void ()** @fptr, align 8
9 ret void
10 }
11
12 ; Function Attrs: norecurse nounwind readnone uwtable
13 define internal void @_ZL3foov() #1 {
14 entry:
15 ret void
16 }
17
18 !llvm.dbg.cu = !{!0}
19 !llvm.module.flags = !{!3}
20 !llvm.ident = !{!31}
21
22 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
23 !1 = !DIFile(filename: "b.cc", directory: "/ssd/llvm/abc/small")
24 !2 = !{}
25 !3 = !{i32 2, !"Debug Info Version", i32 3}
26 !31 = !{!"clang version 5.0.0 (trunk 297016)"}
0 ; Do setup work for all below tests: generate bitcode and combined index
1 ; RUN: opt -module-summary %s -o %t.bc
2 ; RUN: opt -module-summary %p/Inputs/thinlto_samplepgo_icp.ll -o %t2.bc
3 ; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
4
5 ; Checks if calls to static target functions are properly imported and promoted
6 ; by ICP. Note that the GUID in the profile is from the oroginal name.
7 ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS
8 ; IMPORTS: Import _ZL3foov.llvm.0
9 ; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -icp-count-threshold=1 | FileCheck %s --check-prefix=ICALL-PROM
10
11 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
12 target triple = "x86_64-unknown-linux-gnu"
13
14 @fptr = local_unnamed_addr global void ()* null, align 8
15
16 ; Function Attrs: norecurse uwtable
17 define i32 @main() local_unnamed_addr #0 !prof !34 {
18 entry:
19 %0 = load void ()*, void ()** @fptr, align 8
20 ; ICALL-PROM: br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect
21 tail call void %0(), !prof !40
22 ret i32 0
23 }
24
25 !llvm.dbg.cu = !{!0}
26 !llvm.module.flags = !{!3,!4}
27 !llvm.ident = !{!31}
28
29 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
30 !1 = !DIFile(filename: "main.cc", directory: ".")
31 !2 = !{}
32 !3 = !{i32 2, !"Debug Info Version", i32 3}
33 !4 = !{i32 1, !"ProfileSummary", !5}
34 !5 = !{!6, !7, !8, !9, !10, !11, !12, !13}
35 !6 = !{!"ProfileFormat", !"SampleProfile"}
36 !7 = !{!"TotalCount", i64 3003}
37 !8 = !{!"MaxCount", i64 3000}
38 !9 = !{!"MaxInternalCount", i64 0}
39 !10 = !{!"MaxFunctionCount", i64 0}
40 !11 = !{!"NumCounts", i64 3}
41 !12 = !{!"NumFunctions", i64 1}
42 !13 = !{!"DetailedSummary", !14}
43 !14 = !{!15, !16, !17, !18, !19, !20, !20, !21, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
44 !15 = !{i32 10000, i64 3000, i32 1}
45 !16 = !{i32 100000, i64 3000, i32 1}
46 !17 = !{i32 200000, i64 3000, i32 1}
47 !18 = !{i32 300000, i64 3000, i32 1}
48 !19 = !{i32 400000, i64 3000, i32 1}
49 !20 = !{i32 500000, i64 3000, i32 1}
50 !21 = !{i32 600000, i64 3000, i32 1}
51 !22 = !{i32 700000, i64 3000, i32 1}
52 !23 = !{i32 800000, i64 3000, i32 1}
53 !24 = !{i32 900000, i64 3000, i32 1}
54 !25 = !{i32 950000, i64 3000, i32 1}
55 !26 = !{i32 990000, i64 3000, i32 1}
56 !27 = !{i32 999000, i64 3000, i32 1}
57 !28 = !{i32 999900, i64 2, i32 2}
58 !29 = !{i32 999990, i64 2, i32 2}
59 !30 = !{i32 999999, i64 2, i32 2}
60 !31 = !{!"clang version 5.0.0 (trunk 297016)"}
61 !34 = !{!"function_entry_count", i64 1}
62 !40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}