llvm.org GIT mirror llvm / 63ec256
Add a flag to remap manglings when reading profile data information. This can be used to preserve profiling information across codebase changes that have widespread impact on mangled names, but across which most profiling data should still be usable. For example, when switching from libstdc++ to libc++, or from the old libstdc++ ABI to the new ABI, or even from a 32-bit to a 64-bit build. The user can provide a remapping file specifying parts of mangled names that should be treated as equivalent (eg, std::__1 should be treated as equivalent to std::__cxx11), and profile data will be treated as applying to a particular function if its name is equivalent to the name of a function in the profile data under the provided equivalences. See the documentation change for a description of how this is configured. Remapping is supported for both sample-based profiling and instruction profiling. We do not support remapping indirect branch target information, but all other profile data should be remapped appropriately. Support is only added for the new pass manager. If someone wants to also add support for this for the old pass manager, doing so should be straightforward. This is the LLVM side of Clang r344199. Reviewers: davidxl, tejohnson, dlj, erik.pilkington Subscribers: mehdi_amini, steven_wu, dexonsmith, llvm-commits Differential Revision: https://reviews.llvm.org/D51249 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@344200 91177308-0d34-0410-b5e6-96231b3b80d8 Richard Smith 11 months ago
16 changed file(s) with 221 addition(s) and 32 deletion(s). Raw diff Collapse all Expand all
7171
7272 /// Sample PGO profile path.
7373 std::string SampleProfile;
74
75 /// Name remapping file for profile data.
76 std::string ProfileRemapping;
7477
7578 /// The directory to store .dwo files.
7679 std::string DwoDir;
3131 /// A struct capturing PGO tunables.
3232 struct PGOOptions {
3333 PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "",
34 std::string SampleProfileFile = "", bool RunProfileGen = false,
35 bool SamplePGOSupport = false)
34 std::string SampleProfileFile = "",
35 std::string ProfileRemappingFile = "",
36 bool RunProfileGen = false, bool SamplePGOSupport = false)
3637 : ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile),
37 SampleProfileFile(SampleProfileFile), RunProfileGen(RunProfileGen),
38 SampleProfileFile(SampleProfileFile),
39 ProfileRemappingFile(ProfileRemappingFile),
40 RunProfileGen(RunProfileGen),
3841 SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) {
3942 assert((RunProfileGen ||
4043 !SampleProfileFile.empty() ||
4447 std::string ProfileGenFile;
4548 std::string ProfileUseFile;
4649 std::string SampleProfileFile;
50 std::string ProfileRemappingFile;
4751 bool RunProfileGen;
4852 bool SamplePGOSupport;
4953 };
586590 void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
587591 OptimizationLevel Level, bool RunProfileGen,
588592 std::string ProfileGenFile,
589 std::string ProfileUseFile);
593 std::string ProfileUseFile,
594 std::string ProfileRemappingFile);
590595
591596 void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel);
592597
2424 /// The sample profiler data loader pass.
2525 class SampleProfileLoaderPass : public PassInfoMixin {
2626 public:
27 SampleProfileLoaderPass(std::string File = "", bool IsThinLTOPreLink = false)
28 : ProfileFileName(File), IsThinLTOPreLink(IsThinLTOPreLink) {}
27 SampleProfileLoaderPass(std::string File = "", std::string RemappingFile = "",
28 bool IsThinLTOPreLink = false)
29 : ProfileFileName(File), ProfileRemappingFileName(RemappingFile),
30 IsThinLTOPreLink(IsThinLTOPreLink) {}
2931
3032 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
3133
3234 private:
3335 std::string ProfileFileName;
36 std::string ProfileRemappingFileName;
3437 bool IsThinLTOPreLink;
3538 };
3639
3535 /// The profile annotation (profile-instr-use) pass for IR based PGO.
3636 class PGOInstrumentationUse : public PassInfoMixin {
3737 public:
38 PGOInstrumentationUse(std::string Filename = "");
38 PGOInstrumentationUse(std::string Filename = "",
39 std::string RemappingFilename = "");
3940
4041 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
4142
4243 private:
4344 std::string ProfileFileName;
45 std::string ProfileRemappingFileName;
4446 };
4547
4648 /// The indirect function call promotion pass.
262262
263263 if (!Conf.SampleProfile.empty()) {
264264 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
265 if (FileOrErr)
265 if (FileOrErr) {
266266 Hasher.update(FileOrErr.get()->getBuffer());
267
268 if (!Conf.ProfileRemapping.empty()) {
269 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
270 if (FileOrErr)
271 Hasher.update(FileOrErr.get()->getBuffer());
272 }
273 }
267274 }
268275
269276 Key = toHex(Hasher.result());
154154 const ModuleSummaryIndex *ImportSummary) {
155155 Optional PGOOpt;
156156 if (!Conf.SampleProfile.empty())
157 PGOOpt = PGOOptions("", "", Conf.SampleProfile, false, true);
157 PGOOpt = PGOOptions("", "", Conf.SampleProfile, Conf.ProfileRemapping,
158 false, true);
158159
159160 PassBuilder PB(TM, PGOOpt);
160161 AAManager AA;
504504 PassBuilder::OptimizationLevel Level,
505505 bool RunProfileGen,
506506 std::string ProfileGenFile,
507 std::string ProfileUseFile) {
507 std::string ProfileUseFile,
508 std::string ProfileRemappingFile) {
508509 // Generally running simplification passes and the inliner with an high
509510 // threshold results in smaller executables, but there may be cases where
510511 // the size grows, so let's be conservative here and skip this simplification
558559 }
559560
560561 if (!ProfileUseFile.empty())
561 MPM.addPass(PGOInstrumentationUse(ProfileUseFile));
562 MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile));
562563 }
563564
564565 static InlineParams
604605 // Annotate sample profile right after early FPM to ensure freshness of
605606 // the debug info.
606607 MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
608 PGOOpt->ProfileRemappingFile,
607609 Phase == ThinLTOPhase::PreLink));
608610 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
609611 // for the profile annotation to be accurate in the ThinLTO backend.
656658 if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
657659 (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) {
658660 addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
659 PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile);
661 PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile,
662 PGOOpt->ProfileRemappingFile);
660663 MPM.addPass(PGOIndirectCallPromotion(false, false));
661664 }
662665
9595 "sample-profile-file", cl::init(""), cl::value_desc("filename"),
9696 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
9797
98 // The named file contains a set of transformations that may have been applied
99 // to the symbol names between the program from which the sample data was
100 // collected and the current program's symbols.
101 static cl::opt SampleProfileRemappingFile(
102 "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
103 cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
104
98105 static cl::opt SampleProfileMaxPropagateIterations(
99106 "sample-profile-max-propagate-iterations", cl::init(100),
100107 cl::desc("Maximum number of iterations to go through when propagating "
182189 class SampleProfileLoader {
183190 public:
184191 SampleProfileLoader(
185 StringRef Name, bool IsThinLTOPreLink,
192 StringRef Name, StringRef RemapName, bool IsThinLTOPreLink,
186193 std::function GetAssumptionCache,
187194 std::function GetTargetTransformInfo)
188195 : GetAC(std::move(GetAssumptionCache)),
189196 GetTTI(std::move(GetTargetTransformInfo)), Filename(Name),
190 IsThinLTOPreLink(IsThinLTOPreLink) {}
197 RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {}
191198
192199 bool doInitialization(Module &M);
193200 bool runOnModule(Module &M, ModuleAnalysisManager *AM,
281288 /// Name of the profile file to load.
282289 std::string Filename;
283290
291 /// Name of the profile remapping file to load.
292 std::string RemappingFilename;
293
284294 /// Flag indicating whether the profile input loaded successfully.
285295 bool ProfileIsValid = false;
286296
310320
311321 SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile,
312322 bool IsThinLTOPreLink = false)
313 : ModulePass(ID), SampleLoader(Name, IsThinLTOPreLink,
314 [&](Function &F) -> AssumptionCache & {
315 return ACT->getAssumptionCache(F);
316 },
317 [&](Function &F) -> TargetTransformInfo & {
318 return TTIWP->getTTI(F);
319 }) {
323 : ModulePass(ID),
324 SampleLoader(Name, SampleProfileRemappingFile, IsThinLTOPreLink,
325 [&](Function &F) -> AssumptionCache & {
326 return ACT->getAssumptionCache(F);
327 },
328 [&](Function &F) -> TargetTransformInfo & {
329 return TTIWP->getTTI(F);
330 }) {
320331 initializeSampleProfileLoaderLegacyPassPass(
321332 *PassRegistry::getPassRegistry());
322333 }
15141525 Reader = std::move(ReaderOrErr.get());
15151526 Reader->collectFuncsToUse(M);
15161527 ProfileIsValid = (Reader->read() == sampleprof_error::success);
1528
1529 if (!RemappingFilename.empty()) {
1530 // Apply profile remappings to the loaded profile data if requested.
1531 // For now, we only support remapping symbols encoded using the Itanium
1532 // C++ ABI's name mangling scheme.
1533 ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
1534 RemappingFilename, Ctx, std::move(Reader));
1535 if (std::error_code EC = ReaderOrErr.getError()) {
1536 std::string Msg = "Could not open profile remapping file: " + EC.message();
1537 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1538 return false;
1539 }
1540 Reader = std::move(ReaderOrErr.get());
1541 ProfileIsValid = (Reader->read() == sampleprof_error::success);
1542 }
15171543 return true;
15181544 }
15191545
15201546 ModulePass *llvm::createSampleProfileLoaderPass() {
1521 return new SampleProfileLoaderLegacyPass(SampleProfileFile);
1547 return new SampleProfileLoaderLegacyPass();
15221548 }
15231549
15241550 ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
16111637
16121638 SampleProfileLoader SampleLoader(
16131639 ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
1640 ProfileRemappingFileName.empty() ? SampleProfileRemappingFile
1641 : ProfileRemappingFileName,
16141642 IsThinLTOPreLink, GetAssumptionCache, GetTTI);
16151643
16161644 SampleLoader.doInitialization(M);
140140 cl::value_desc("filename"),
141141 cl::desc("Specify the path of profile data file. This is"
142142 "mainly for test purpose."));
143 static cl::opt PGOTestProfileRemappingFile(
144 "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden,
145 cl::value_desc("filename"),
146 cl::desc("Specify the path of profile remapping file. This is mainly for "
147 "test purpose."));
143148
144149 // Command line option to disable value profiling. The default is false:
145150 // i.e. value profiling is enabled by default. This is for debug purpose.
14281433 }
14291434
14301435 static bool annotateAllFunctions(
1431 Module &M, StringRef ProfileFileName,
1436 Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
14321437 function_ref LookupBPI,
14331438 function_ref LookupBFI) {
14341439 LLVM_DEBUG(dbgs() << "Read in profile counters: ");
14351440 auto &Ctx = M.getContext();
14361441 // Read the counter array from file.
1437 auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName);
1442 auto ReaderOrErr =
1443 IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName);
14381444 if (Error E = ReaderOrErr.takeError()) {
14391445 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {
14401446 Ctx.diagnose(
15281534 return true;
15291535 }
15301536
1531 PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename)
1532 : ProfileFileName(std::move(Filename)) {
1537 PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename,
1538 std::string RemappingFilename)
1539 : ProfileFileName(std::move(Filename)),
1540 ProfileRemappingFileName(std::move(RemappingFilename)) {
15331541 if (!PGOTestProfileFile.empty())
15341542 ProfileFileName = PGOTestProfileFile;
1543 if (!PGOTestProfileRemappingFile.empty())
1544 ProfileRemappingFileName = PGOTestProfileRemappingFile;
15351545 }
15361546
15371547 PreservedAnalyses PGOInstrumentationUse::run(Module &M,
15461556 return &FAM.getResult(F);
15471557 };
15481558
1549 if (!annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI))
1559 if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
1560 LookupBPI, LookupBFI))
15501561 return PreservedAnalyses::all();
15511562
15521563 return PreservedAnalyses::none();
15631574 return &this->getAnalysis(F).getBFI();
15641575 };
15651576
1566 return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI);
1577 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI);
15671578 }
15681579
15691580 static std::string getSimpleNodeName(const BasicBlock *Node) {
0 # foo:: and foo::detail:: are equivalent
1 name 3foo N3foo6detailE
2
3 # foo::qux and foo::quux are equivalent
4 type N3foo3quxE N3foo4quuxE
5
6 # N::X and M::X are equivalent
7 name N1N1XE N1M1XE
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 _ZN3foo3barERKN1N1XINS_4quuxEEE
3 25571299074
4 2
5 3
6 2
7
0 ; RUN: llvm-profdata merge %S/Inputs/remap.proftext -o %t.profdata
1 ; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-test-profile-remapping-file=%S/Inputs/remap.map -S | FileCheck %s --check-prefix=USE
2
3 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
4 target triple = "x86_64-unknown-linux-gnu"
5
6 define i32 @_ZN3foo3barERKN1M1XINS_6detail3quxEEE(i32 %i) {
7 ; USE-LABEL: @_ZN3foo3barERKN1M1XINS_6detail3quxEEE
8 ; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]]
9 entry:
10 %cmp = icmp sgt i32 %i, 0
11 br i1 %cmp, label %if.then, label %if.end
12 ; USE: br i1 %cmp, label %if.then, label %if.end
13 ; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
14
15 if.then:
16 %add = add nsw i32 %i, 2
17 br label %if.end
18
19 if.end:
20 %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
21 ret i32 %retv
22 }
23
24 ; USE-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
25 ; USE-DAG: {{![0-9]+}} = !{!"DetailedSummary", {{![0-9]+}}}
26 ; USE-DAG: ![[FUNC_ENTRY_COUNT]] = !{!"function_entry_count", i64 3}
27 ; USE-DAG: ![[BW_ENTRY]] = !{!"branch_weights", i32 2, i32 1}
0 # foo:: and foo::detail:: are equivalent
1 name 3foo N3foo6detailE
2
3 # foo::qux and foo::quux are equivalent
4 type N3foo3quxE N3foo4quuxE
5
6 # N::X and M::X are equivalent
7 name N1N1XE N1M1XE
0 _ZN3foo3barERKN1N1XINS_4quuxEEE:15680:2500
1 1: 2500
2 4: 1000
3 5: 1000
4 6: 800
5 7: 500
6 9: 10226
7 10: 2243
8 16: 0
9 18: 0
0 ; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s
1
2 ; Reduced from branch.ll
3
4 declare i1 @foo()
5
6 define void @_ZN3foo3barERKN1M1XINS_6detail3quxEEE() !dbg !2 {
7 ; CHECK: Printing analysis 'Branch Probability Analysis' for function '_ZN3foo3barERKN1M1XINS_6detail3quxEEE':
8
9 entry:
10 %cmp = call i1 @foo(), !dbg !6
11 br i1 %cmp, label %if.then, label %if.end
12 ; CHECK: edge entry -> if.then probability is 0x4ccf6b16 / 0x80000000 = 60.01%
13 ; CHECK: edge entry -> if.end probability is 0x333094ea / 0x80000000 = 39.99%
14
15 if.then:
16 br label %return
17
18 if.end:
19 %cmp1 = call i1 @foo(), !dbg !7
20 br i1 %cmp1, label %if.then.2, label %if.else
21 ; CHECK: edge if.end -> if.then.2 probability is 0x6652c748 / 0x80000000 = 79.94%
22 ; CHECK: edge if.end -> if.else probability is 0x19ad38b8 / 0x80000000 = 20.06%
23
24 if.then.2:
25 call i1 @foo(), !dbg !8
26 br label %for.cond
27
28 for.cond:
29 %cmp5 = call i1 @foo()
30 br i1 %cmp5, label %for.body, label %for.end, !prof !9
31 ; CHECK: edge for.cond -> for.body probability is 0x73333333 / 0x80000000 = 90.00%
32 ; CHECK: edge for.cond -> for.end probability is 0x0ccccccd / 0x80000000 = 10.00%
33
34 for.body:
35 br label %for.cond
36
37 for.end:
38 br label %return
39
40 if.else:
41 br label %return
42
43 return:
44 ret void
45 }
46
47 !llvm.dbg.cu = !{!0}
48 !llvm.module.flags = !{!4, !5}
49
50 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "foo++", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !{}, retainedTypes: !{})
51 !1 = !DIFile(filename: "test.cc", directory: "/foo/bar")
52 !2 = distinct !DISubprogram(name: "_ZN3foo3barERKN1M1XINS_6detail3quxEEE", scope: !1, file: !1, line: 4, type: !3, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !{})
53 !3 = !DISubroutineType(types: !{})
54 !4 = !{i32 2, !"Dwarf Version", i32 4}
55 !5 = !{i32 2, !"Debug Info Version", i32 3}
56 !6 = !DILocation(line: 5, column: 8, scope: !2)
57 !7 = !DILocation(line: 8, column: 6, scope: !2)
58 !8 = !DILocation(line: 10, column: 11, scope: !2)
59 !9 = !{!"branch_weights", i32 90, i32 10}
107107 "Use sampled profile to guide PGO.")));
108108 static cl::opt ProfileFile(
109109 "profile-file", cl::desc("Path to the profile."), cl::Hidden);
110 static cl::opt
111 ProfileRemappingFile("profile-remapping-file",
112 cl::desc("Path to the profile remapping file."),
113 cl::Hidden);
110114 static cl::opt DebugInfoForProfiling(
111115 "new-pm-debug-info-for-profiling", cl::init(false), cl::Hidden,
112116 cl::desc("Emit special debug info to enable PGO profile generation."));
199203 Optional P;
200204 switch (PGOKindFlag) {
201205 case InstrGen:
202 P = PGOOptions(ProfileFile, "", "", true);
206 P = PGOOptions(ProfileFile, "", "", "", true);
203207 break;
204208 case InstrUse:
205 P = PGOOptions("", ProfileFile, "", false);
209 P = PGOOptions("", ProfileFile, "", ProfileRemappingFile, false);
206210 break;
207211 case SampleUse:
208 P = PGOOptions("", "", ProfileFile, false);
212 P = PGOOptions("", "", ProfileFile, ProfileRemappingFile, false);
209213 break;
210214 case NoPGO:
211215 if (DebugInfoForProfiling)
212 P = PGOOptions("", "", "", false, true);
216 P = PGOOptions("", "", "", "", false, true);
213217 else
214218 P = None;
215219 }