llvm.org GIT mirror llvm / d97a419
Revert "[PGO] Context sensitive PGO (part 1)" This reverts commit r354930, it was causing UBSan failures. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@354953 91177308-0d34-0410-b5e6-96231b3b80d8 Vlad Tsyrklevich 1 year, 9 months ago
14 changed file(s) with 75 addition(s) and 309 deletion(s). Raw diff Collapse all Expand all
298298 void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&);
299299 void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&);
300300 void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&);
301 void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&);
302301 void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&);
303302 void initializePHIEliminationPass(PassRegistry&);
304303 void initializePartialInlinerLegacyPassPass(PassRegistry&);
5454 /// Disable entirely the optimizer, including importing for ThinLTO
5555 bool CodeGenOnly = false;
5656
57 /// Run PGO context sensitive IR instrumentation.
58 bool RunCSIRInstr = false;
59
6057 /// If this field is set, the set of passes run in the middle-end optimizer
6158 /// will be the one specified by the string. Only works with the new pass
6259 /// manager as the old one doesn't have this ability.
7471 /// Setting this field will replace unspecified target triples in input files
7572 /// with this triple.
7673 std::string DefaultTriple;
77
78 /// Context Sensitive PGO profile path.
79 std::string CSIRProfile;
8074
8175 /// Sample PGO profile path.
8276 std::string SampleProfile;
101101 (void) llvm::createGCOVProfilerPass();
102102 (void) llvm::createPGOInstrumentationGenLegacyPass();
103103 (void) llvm::createPGOInstrumentationUseLegacyPass();
104 (void) llvm::createPGOInstrumentationGenCreateVarLegacyPass();
105104 (void) llvm::createPGOIndirectCallPromotionLegacyPass();
106105 (void) llvm::createPGOMemOPSizeOptLegacyPass();
107106 (void) llvm::createInstrProfilingLegacyPass();
766766 StringRef Name;
767767 uint64_t Hash;
768768
769 // We reserve this bit as the flag for context sensitive profile record.
770 static const int CS_FLAG_IN_FUNC_HASH = 60;
771
772769 NamedInstrProfRecord() = default;
773770 NamedInstrProfRecord(StringRef Name, uint64_t Hash,
774771 std::vector Counts)
775772 : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
776
777 static bool hasCSFlagInHash(uint64_t FuncHash) {
778 return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
779 }
780 static void setCSFlagInHash(uint64_t &FuncHash) {
781 FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
782 }
783773 };
784774
785775 uint32_t InstrProfRecord::getNumValueKinds() const {
10131003 // from control data struct is changed from raw pointer to Name's MD5 value.
10141004 // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
10151005 // raw header.
1016 // Version 5: Bit 60 of FuncHash is reserved for the flag for the context
1017 // sensitive records.
10181006 const uint64_t Version = INSTR_PROF_RAW_VERSION;
10191007
10201008 template inline uint64_t getMagic();
10511039 void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
10521040 int64_t &RangeLast);
10531041
1054 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1055 // aware this is an ir_level profile so it can set the version flag.
1056 void createIRLevelProfileFlagVar(Module &M, bool IsCS);
1057
10581042 // Create the variable for the profile file name.
10591043 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
10601044
634634 * version for other variants of profile. We set the lowest bit of the upper 8
635635 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
636636 * generated profile, and 0 if this is a Clang FE generated profile.
637 * 1 in bit 57 indicates there are context-sensitive records in the profile.
638637 */
639638 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
640639 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
641640 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
642 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
643641 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
644642 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
645643
3434 class InstrProfiling : public PassInfoMixin {
3535 public:
3636 InstrProfiling() = default;
37 InstrProfiling(const InstrProfOptions &Options, bool IsCS)
38 : Options(Options), IsCS(IsCS) {}
37 InstrProfiling(const InstrProfOptions &Options) : Options(Options) {}
3938
4039 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
4140 bool run(Module &M, const TargetLibraryInfo &TLI);
5958 std::vector ReferencedNames;
6059 GlobalVariable *NamesVar;
6160 size_t NamesSize;
62
63 // Is this lowering for the context-sensitive instrumentation.
64 bool IsCS;
6561
6662 // vector of counter load/store pairs to be register promoted.
6763 std::vector PromotionCandidates;
1616
1717 #include "llvm/ADT/ArrayRef.h"
1818 #include "llvm/IR/PassManager.h"
19 #include "llvm/ProfileData/InstrProf.h"
2019 #include
2120 #include
2221
2726 class Module;
2827
2928 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
30 // We use this pass to create COMDAT profile variables for context
31 // sensitive PGO (CSPGO). The reason to have a pass for this is CSPGO
32 // can be run after LTO/ThinLTO linking. Lld linker needs to see
33 // all the COMDAT variables before linking. So we have this pass
34 // always run before linking for CSPGO.
35 class PGOInstrumentationGenCreateVar
36 : public PassInfoMixin {
37 public:
38 PGOInstrumentationGenCreateVar(std::string CSInstrName = "")
39 : CSInstrName(CSInstrName) {}
40 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) {
41 createProfileFileNameVar(M, CSInstrName);
42 createIRLevelProfileFlagVar(M, /* IsCS */ true);
43 return PreservedAnalyses::all();
44 }
45
46 private:
47 std::string CSInstrName;
48 };
49
50 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
5129 class PGOInstrumentationGen : public PassInfoMixin {
5230 public:
53 PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {}
5431 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
55
56 private:
57 // If this is a context sensitive instrumentation.
58 bool IsCS;
5932 };
6033
6134 /// The profile annotation (profile-instr-use) pass for IR based PGO.
6235 class PGOInstrumentationUse : public PassInfoMixin {
6336 public:
6437 PGOInstrumentationUse(std::string Filename = "",
65 std::string RemappingFilename = "", bool IsCS = false);
38 std::string RemappingFilename = "");
6639
6740 PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
6841
6942 private:
7043 std::string ProfileFileName;
7144 std::string ProfileRemappingFileName;
72 // If this is a context sensitive instrumentation.
73 bool IsCS;
7445 };
7546
7647 /// The indirect function call promotion pass.
8686 ModulePass *createGCOVProfilerPass(const GCOVOptions &Options =
8787 GCOVOptions::getDefault());
8888
89 // PGO Instrumention. Parameter IsCS indicates if this is the context senstive
90 // instrumentation.
91 ModulePass *createPGOInstrumentationGenLegacyPass(bool IsCS = false);
89 // PGO Instrumention
90 ModulePass *createPGOInstrumentationGenLegacyPass();
9291 ModulePass *
93 createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""),
94 bool IsCS = false);
95 ModulePass *createPGOInstrumentationGenCreateVarLegacyPass(
96 StringRef CSInstrName = StringRef(""));
92 createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""));
9793 ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false,
9894 bool SamplePGO = false);
9995 FunctionPass *createPGOMemOPSizeOptLegacyPass();
135131 // Use atomic profile counter increments.
136132 bool Atomic = false;
137133
138 // Use BFI to guide register promotion
139 bool UseBFIInPromotion = false;
140
141134 // Name of the profile file to use as output
142135 std::string InstrProfileOutput;
143136
144137 InstrProfOptions() = default;
145138 };
146139
147 /// Insert frontend instrumentation based profiling. Parameter IsCS indicates if
148 // this is the context senstive instrumentation.
140 /// Insert frontend instrumentation based profiling.
149141 ModulePass *createInstrProfilingLegacyPass(
150 const InstrProfOptions &Options = InstrProfOptions(), bool IsCS = false);
142 const InstrProfOptions &Options = InstrProfOptions());
151143
152144 FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false,
153145 bool Recover = false);
568568 if (!ProfileGenFile.empty())
569569 Options.InstrProfileOutput = ProfileGenFile;
570570 Options.DoCounterPromotion = true;
571 Options.UseBFIInPromotion = false;
572 MPM.addPass(InstrProfiling(Options, false));
571 MPM.addPass(InstrProfiling(Options));
573572 }
574573
575574 if (!ProfileUseFile.empty())
10101010 assert(RangeLast >= RangeStart);
10111011 }
10121012
1013 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1014 // aware this is an ir_level profile so it can set the version flag.
1015 void createIRLevelProfileFlagVar(Module &M, bool IsCS) {
1016 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1017 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1018 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
1019 if (IsCS)
1020 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
1021 auto IRLevelVersionVariable = new GlobalVariable(
1022 M, IntTy64, true, GlobalValue::WeakAnyLinkage,
1023 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
1024 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1025 Triple TT(M.getTargetTriple());
1026 if (TT.supportsCOMDAT()) {
1027 IRLevelVersionVariable->setLinkage(GlobalValue::ExternalLinkage);
1028 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
1029 }
1030 }
1031
10321013 // Create the variable for the profile file name.
10331014 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
10341015 if (InstrProfileOutput.empty())
1717 #include "llvm/ADT/StringRef.h"
1818 #include "llvm/ADT/Triple.h"
1919 #include "llvm/ADT/Twine.h"
20 #include "llvm/Analysis/BlockFrequencyInfo.h"
21 #include "llvm/Analysis/BranchProbabilityInfo.h"
2220 #include "llvm/Analysis/LoopInfo.h"
2321 #include "llvm/Analysis/TargetLibraryInfo.h"
2422 #include "llvm/IR/Attributes.h"
148146 static char ID;
149147
150148 InstrProfilingLegacyPass() : ModulePass(ID) {}
151 InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS)
152 : ModulePass(ID), InstrProf(Options, IsCS) {}
149 InstrProfilingLegacyPass(const InstrProfOptions &Options)
150 : ModulePass(ID), InstrProf(Options) {}
153151
154152 StringRef getPassName() const override {
155153 return "Frontend instrumentation-based coverage lowering";
233231 public:
234232 PGOCounterPromoter(
235233 DenseMap> &LoopToCands,
236 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
234 Loop &CurLoop, LoopInfo &LI)
237235 : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
238 LI(LI), BFI(BFI) {
236 LI(LI) {
239237
240238 SmallVector LoopExitBlocks;
241239 SmallPtrSet BlockSet;
263261 SmallVector NewPHIs;
264262 SSAUpdater SSA(&NewPHIs);
265263 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
266
267 // If BFI is set, we will use it to guide the promotions.
268 if (BFI) {
269 auto *BB = Cand.first->getParent();
270 auto InstrCount = BFI->getBlockProfileCount(BB);
271 if (!InstrCount)
272 continue;
273 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
274 // If the average loop trip count is not greater than 1.5, we skip
275 // promotion.
276 if (PreheaderCount &&
277 (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
278 continue;
279 }
280264
281265 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
282266 L.getLoopPreheader(), ExitBlocks,
327311
328312 SmallVector ExitingBlocks;
329313 LP->getExitingBlocks(ExitingBlocks);
330
331 // If BFI is set, we do more aggressive promotions based on BFI.
332 if (BFI)
333 return (unsigned)-1;
334
335314 // Not considierered speculative.
336315 if (ExitingBlocks.size() == 1)
337316 return MaxNumOfPromotionsPerLoop;
363342 SmallVector InsertPts;
364343 Loop &L;
365344 LoopInfo &LI;
366 BlockFrequencyInfo *BFI;
367345 };
368346
369347 } // end anonymous namespace
386364 "Frontend instrumentation-based coverage lowering.", false, false)
387365
388366 ModulePass *
389 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
390 bool IsCS) {
391 return new InstrProfilingLegacyPass(Options, IsCS);
367 llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options) {
368 return new InstrProfilingLegacyPass(Options);
392369 }
393370
394371 static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
437414 LoopInfo LI(DT);
438415 DenseMap> LoopPromotionCandidates;
439416
440 std::unique_ptr BFI;
441 if (Options.UseBFIInPromotion) {
442 std::unique_ptr BPI;
443 BPI.reset(new BranchProbabilityInfo(*F, LI, TLI));
444 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
445 }
446
447417 for (const auto &LoadStore : PromotionCandidates) {
448418 auto *CounterLoad = LoadStore.first;
449419 auto *CounterStore = LoadStore.second;
459429 // Do a post-order traversal of the loops so that counter updates can be
460430 // iteratively hoisted outside the loop nest.
461431 for (auto *Loop : llvm::reverse(Loops)) {
462 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
432 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI);
463433 Promoter.run(&TotalCountersPromoted);
464434 }
465435 }
710680 // Don't do this for Darwin. compiler-rt uses linker magic.
711681 if (TT.isOSDarwin())
712682 return false;
683
713684 // Use linker script magic to get data/cnts/name start/end.
714685 if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
715686 TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
1013984 }
1014985
1015986 void InstrProfiling::emitInitialization() {
1016 // Create ProfileFileName variable. Don't don't this for the
1017 // context-sensitive instrumentation lowering: This lowering is after
1018 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1019 // have already create the variable before LTO/ThinLTO linking.
1020 if (!IsCS)
1021 createProfileFileNameVar(*M, Options.InstrProfileOutput);
987 // Create variable for profile name.
988 createProfileFileNameVar(*M, Options.InstrProfileOutput);
1022989 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1023990 if (!RegisterF)
1024991 return;
6464 #include "llvm/Analysis/IndirectCallVisitor.h"
6565 #include "llvm/Analysis/LoopInfo.h"
6666 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
67 #include "llvm/Analysis/ProfileSummaryInfo.h"
6867 #include "llvm/IR/Attributes.h"
6968 #include "llvm/IR/BasicBlock.h"
7069 #include "llvm/IR/CFG.h"
132131 STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
133132 STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
134133 STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");
135 STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.");
136 STATISTIC(NumOfCSPGOSelectInsts,
137 "Number of select instruction instrumented in CSPGO.");
138 STATISTIC(NumOfCSPGOMemIntrinsics,
139 "Number of mem intrinsics instrumented in CSPGO.");
140 STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO.");
141 STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO.");
142 STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.");
143 STATISTIC(NumOfCSPGOFunc,
144 "Number of functions having valid profile counts in CSPGO.");
145 STATISTIC(NumOfCSPGOMismatch,
146 "Number of functions having mismatch profile in CSPGO.");
147 STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.");
148134
149135 // Command line option to specify the file to read profile from. This is
150136 // mainly used for testing.
396382 public:
397383 static char ID;
398384
399 PGOInstrumentationGenLegacyPass(bool IsCS = false)
400 : ModulePass(ID), IsCS(IsCS) {
385 PGOInstrumentationGenLegacyPass() : ModulePass(ID) {
401386 initializePGOInstrumentationGenLegacyPassPass(
402387 *PassRegistry::getPassRegistry());
403388 }
405390 StringRef getPassName() const override { return "PGOInstrumentationGenPass"; }
406391
407392 private:
408 // Is this is context-sensitive instrumentation.
409 bool IsCS;
410393 bool runOnModule(Module &M) override;
411394
412395 void getAnalysisUsage(AnalysisUsage &AU) const override {
419402 static char ID;
420403
421404 // Provide the profile filename as the parameter.
422 PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false)
423 : ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) {
405 PGOInstrumentationUseLegacyPass(std::string Filename = "")
406 : ModulePass(ID), ProfileFileName(std::move(Filename)) {
424407 if (!PGOTestProfileFile.empty())
425408 ProfileFileName = PGOTestProfileFile;
426409 initializePGOInstrumentationUseLegacyPassPass(
431414
432415 private:
433416 std::string ProfileFileName;
434 // Is this is context-sensitive instrumentation use.
435 bool IsCS;
436417
437418 bool runOnModule(Module &M) override;
438419
439420 void getAnalysisUsage(AnalysisUsage &AU) const override {
440 AU.addRequired();
441421 AU.addRequired();
442422 }
443 };
444
445 class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass {
446 public:
447 static char ID;
448 StringRef getPassName() const override {
449 return "PGOInstrumentationGenCreateVarPass";
450 }
451 PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "")
452 : ModulePass(ID), InstrProfileOutput(CSInstrName) {
453 initializePGOInstrumentationGenCreateVarLegacyPassPass(
454 *PassRegistry::getPassRegistry());
455 }
456
457 private:
458 bool runOnModule(Module &M) override {
459 createProfileFileNameVar(M, InstrProfileOutput);
460 createIRLevelProfileFlagVar(M, true);
461 return false;
462 }
463 std::string InstrProfileOutput;
464423 };
465424
466425 } // end anonymous namespace
474433 INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
475434 "PGO instrumentation.", false, false)
476435
477 ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) {
478 return new PGOInstrumentationGenLegacyPass(IsCS);
436 ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
437 return new PGOInstrumentationGenLegacyPass();
479438 }
480439
481440 char PGOInstrumentationUseLegacyPass::ID = 0;
484443 "Read PGO instrumentation profile.", false, false)
485444 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
486445 INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
487 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
488446 INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
489447 "Read PGO instrumentation profile.", false, false)
490448
491 ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename,
492 bool IsCS) {
493 return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS);
494 }
495
496 char PGOInstrumentationGenCreateVarLegacyPass::ID = 0;
497
498 INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass,
499 "pgo-instr-gen-create-var",
500 "Create PGO instrumentation version variable for CSPGO.", false,
501 false)
502
503 ModulePass *
504 llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) {
505 return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName);
449 ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
450 return new PGOInstrumentationUseLegacyPass(Filename.str());
506451 }
507452
508453 namespace {
550495 private:
551496 Function &F;
552497
553 // Is this is context-sensitive instrumentation.
554 bool IsCS;
555
556498 // A map that stores the Comdat group in function F.
557499 std::unordered_multimap &ComdatMembers;
558500
592534 Function &Func,
593535 std::unordered_multimap &ComdatMembers,
594536 bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
595 BlockFrequencyInfo *BFI = nullptr, bool IsCS = false)
596 : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers),
597 ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func),
598 MST(F, BPI, BFI) {
537 BlockFrequencyInfo *BFI = nullptr)
538 : F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
539 SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) {
599540 // This should be done before CFG hash computation.
600541 SIVisitor.countSelects(Func);
601542 MIVisitor.countMemIntrinsics(Func);
602 if (!IsCS) {
603 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
604 NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
605 NumOfPGOBB += MST.BBInfos.size();
606 ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
607 } else {
608 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
609 NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
610 NumOfCSPGOBB += MST.BBInfos.size();
611 }
543 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
544 NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
545 ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
612546 ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func);
613547
614548 FuncName = getPGOFuncName(F);
617551 renameComdatFunction();
618552 LLVM_DEBUG(dumpInfo("after CFGMST"));
619553
554 NumOfPGOBB += MST.BBInfos.size();
620555 for (auto &E : MST.AllEdges) {
621556 if (E->Removed)
622557 continue;
623 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
558 NumOfPGOEdge++;
624559 if (!E->InMST)
625 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
560 NumOfPGOInstrument++;
626561 }
627562
628563 if (CreateGlobalVar)
661596 }
662597 }
663598 JC.update(Indexes);
664
665 // Hash format for context sensitive profile. Reserve 4 bits for other
666 // information.
667599 FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
668600 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
669 //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
670601 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
671 // Reserve bit 60-63 for other information purpose.
672 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
673 if (IsCS)
674 NamedInstrProfRecord::setCSFlagInHash(FunctionHash);
675602 LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"
676603 << " CRC = " << JC.getCRC()
677604 << ", Selects = " << SIVisitor.getNumOfSelectInsts()
777704
778705 // For a critical edge, we have to split. Instrument the newly
779706 // created BB.
780 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
707 NumOfPGOSplit++;
781708 LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index
782709 << " --> " << getBBInfo(DestBB).Index << "\n");
783710 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
792719 // Critical edges will be split.
793720 static void instrumentOneFunc(
794721 Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI,
795 std::unordered_multimap &ComdatMembers,
796 bool IsCS) {
722 std::unordered_multimap &ComdatMembers) {
797723 // Split indirectbr critical edges here before computing the MST rather than
798724 // later in getInstrBB() to avoid invalidating it.
799725 SplitIndirectBrCriticalEdges(F, BPI, BFI);
800
801726 FuncPGOInstrumentation FuncInfo(F, ComdatMembers, true, BPI,
802 BFI, IsCS);
727 BFI);
803728 unsigned NumCounters = FuncInfo.getNumCounters();
804729
805730 uint32_t I = 0;
926851 PGOUseFunc(Function &Func, Module *Modu,
927852 std::unordered_multimap &ComdatMembers,
928853 BranchProbabilityInfo *BPI = nullptr,
929 BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false)
854 BlockFrequencyInfo *BFIin = nullptr)
930855 : F(Func), M(Modu), BFI(BFIin),
931 FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS),
932 FreqAttr(FFA_Normal), IsCS(IsCS) {}
856 FuncInfo(Func, ComdatMembers, false, BPI, BFIin),
857 FreqAttr(FFA_Normal) {}
933858
934859 // Read counts for the instrumented BB from profile.
935860 bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros);
1001926
1002927 // Function hotness info derived from profile.
1003928 FuncFreqAttr FreqAttr;
1004
1005 // Is to use the context sensitive profile.
1006 bool IsCS;
1007929
1008930 // Find the Instrumented BB and set the value.
1009931 void setInstrumentedCounts(const std::vector &CountFromProfile);
10981020 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
10991021 auto Err = IPE.get();
11001022 bool SkipWarning = false;
1101 LLVM_DEBUG(dbgs() << "Error in reading profile for Func "
1102 << FuncInfo.FuncName << ": ");
11031023 if (Err == instrprof_error::unknown_function) {
1104 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1024 NumOfPGOMissing++;
11051025 SkipWarning = !PGOWarnMissing;
1106 LLVM_DEBUG(dbgs() << "unknown function");
11071026 } else if (Err == instrprof_error::hash_mismatch ||
11081027 Err == instrprof_error::malformed) {
1109 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1028 NumOfPGOMismatch++;
11101029 SkipWarning =
11111030 NoPGOWarnMismatch ||
11121031 (NoPGOWarnMismatchComdat &&
11131032 (F.hasComdat() ||
11141033 F.getLinkage() == GlobalValue::AvailableExternallyLinkage));
1115 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");
11161034 }
11171035
1118 LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n");
11191036 if (SkipWarning)
11201037 return;
11211038
1122 std::string Msg = IPE.message() + std::string(" ") + F.getName().str() +
1123 std::string(" Hash = ") +
1124 std::to_string(FuncInfo.FunctionHash);
1125
1039 std::string Msg = IPE.message() + std::string(" ") + F.getName().str();
11261040 Ctx.diagnose(
11271041 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
11281042 });
11311045 ProfileRecord = std::move(Result.get());
11321046 std::vector &CountFromProfile = ProfileRecord.Counts;
11331047
1134 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1048 NumOfPGOFunc++;
11351049 LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
11361050 uint64_t ValueSum = 0;
11371051 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
11461060 getBBInfo(nullptr).UnknownCountInEdge = 2;
11471061
11481062 setInstrumentedCounts(CountFromProfile);
1149 #if 0
1150 ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
1151 #else
11521063 ProgramMaxCount = PGOReader->getMaximumFunctionCount();
1153 #endif
11541064 return true;
11551065 }
11561066
12551165 // Assign the scaled count values to the BB with multiple out edges.
12561166 void PGOUseFunc::setBranchWeights() {
12571167 // Generate MD_prof metadata for every branch instruction.
1258 LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName()
1259 << " IsCS=" << IsCS << "\n");
1168 LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n");
12601169 for (auto &BB : F) {
12611170 Instruction *TI = BB.getTerminator();
12621171 if (TI->getNumSuccessors() < 2)
12641173 if (!(isa(TI) || isa(TI) ||
12651174 isa(TI)))
12661175 continue;
1267
12681176 if (getBBInfo(&BB).CountValue == 0)
12691177 continue;
12701178
14421350 }
14431351 }
14441352
1353 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1354 // aware this is an ir_level profile so it can set the version flag.
1355 static void createIRLevelProfileFlagVariable(Module &M) {
1356 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1357 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
1358 auto IRLevelVersionVariable = new GlobalVariable(
1359 M, IntTy64, true, GlobalVariable::ExternalLinkage,
1360 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
1361 INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1362 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1363 Triple TT(M.getTargetTriple());
1364 if (!TT.supportsCOMDAT())
1365 IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage);
1366 else
1367 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(
1368 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR))));
1369 }
1370
14451371 // Collect the set of members for each Comdat in module M and store
14461372 // in ComdatMembers.
14471373 static void collectComdatMembers(
14621388
14631389 static bool InstrumentAllFunctions(
14641390 Module &M, function_ref LookupBPI,
1465 function_ref LookupBFI, bool IsCS) {
1466 // For the context-sensitve instrumentation, we should have a separated pass
1467 // (before LTO/ThinLTO linking) to create these variables.
1468 if (!IsCS)
1469 createIRLevelProfileFlagVar(M, /* IsCS */ false);
1391 function_ref LookupBFI) {
1392 createIRLevelProfileFlagVariable(M);
14701393 std::unordered_multimap ComdatMembers;
14711394 collectComdatMembers(M, ComdatMembers);
14721395
14751398 continue;
14761399 auto *BPI = LookupBPI(F);
14771400 auto *BFI = LookupBFI(F);
1478 instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS);
1401 instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers);
14791402 }
14801403 return true;
14811404 }
14901413 auto LookupBFI = [this](Function &F) {
14911414 return &this->getAnalysis(F).getBFI();
14921415 };
1493 return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS);
1416 return InstrumentAllFunctions(M, LookupBPI, LookupBFI);
14941417 }
14951418
14961419 PreservedAnalyses PGOInstrumentationGen::run(Module &M,
15041427 return &FAM.getResult(F);
15051428 };
15061429
1507 if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS))
1430 if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI))
15081431 return PreservedAnalyses::all();
15091432
15101433 return PreservedAnalyses::none();
15131436 static bool annotateAllFunctions(
15141437 Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
15151438 function_ref LookupBPI,
1516 function_ref LookupBFI, bool IsCS) {
1439 function_ref LookupBFI) {
15171440 LLVM_DEBUG(dbgs() << "Read in profile counters: ");
15181441 auto &Ctx = M.getContext();
15191442 // Read the counter array from file.
15341457 StringRef("Cannot get PGOReader")));
15351458 return false;
15361459 }
1537 #if 0
1538 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
1539 return false;
1540 #endif
1541
15421460 // TODO: might need to change the warning once the clang option is finalized.
15431461 if (!PGOReader->isIRLevelProfile()) {
15441462 Ctx.diagnose(DiagnosticInfoPGOProfile(
15581476 // Split indirectbr critical edges here before computing the MST rather than
15591477 // later in getInstrBB() to avoid invalidating it.
15601478 SplitIndirectBrCriticalEdges(F, BPI, BFI);
1561 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS);
1479 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI);
15621480 bool AllZeros = false;
15631481 if (!Func.readCounters(PGOReader.get(), AllZeros))
15641482 continue;
16061524 }
16071525 }
16081526 }
1609 #if 0
1610 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
1611 IsCS ? ProfileSummary::PSK_CSInstr
1612 : ProfileSummary::PSK_Instr);
1613 #else
16141527 M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
1615 #endif
1616
16171528 // Set function hotness attribute from the profile.
16181529 // We have to apply these attributes at the end because their presence
16191530 // can affect the BranchProbabilityInfo of any callers, resulting in an
16321543 }
16331544
16341545 PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename,
1635 std::string RemappingFilename,
1636 bool IsCS)
1546 std::string RemappingFilename)
16371547 : ProfileFileName(std::move(Filename)),
1638 ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) {
1548 ProfileRemappingFileName(std::move(RemappingFilename)) {
16391549 if (!PGOTestProfileFile.empty())
16401550 ProfileFileName = PGOTestProfileFile;
16411551 if (!PGOTestProfileRemappingFile.empty())
16551565 };
16561566
16571567 if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
1658 LookupBPI, LookupBFI, IsCS))
1568 LookupBPI, LookupBFI))
16591569 return PreservedAnalyses::all();
16601570
16611571 return PreservedAnalyses::none();
16721582 return &this->getAnalysis(F).getBFI();
16731583 };
16741584
1675 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI,
1676 IsCS);
1585 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI);
16771586 }
16781587
16791588 static std::string getSimpleNodeName(const BasicBlock *Node) {
208208 static std::string OptRemarksFilename;
209209 static bool OptRemarksWithHotness = false;
210210
211 // Context sensitive PGO options.
212 static std::string cs_profile_path;
213 static bool cs_pgo_gen = false;
214
215211 static void process_plugin_option(const char *opt_)
216212 {
217213 if (opt_ == nullptr)
271267 } else if (opt == "disable-verify") {
272268 DisableVerify = true;
273269 } else if (opt.startswith("sample-profile=")) {
274 sample_profile = opt.substr(strlen("sample-profile="));
275 } else if (opt == "cs-profile-generate") {
276 cs_pgo_gen = true;
277 } else if (opt.startswith("cs-profile-path=")) {
278 cs_profile_path = opt.substr(strlen("cs-profile-path="));
270 sample_profile= opt.substr(strlen("sample-profile="));
279271 } else if (opt == "new-pass-manager") {
280272 new_pass_manager = true;
281273 } else if (opt == "debug-pass-manager") {
898890
899891 if (!options::sample_profile.empty())
900892 Conf.SampleProfile = options::sample_profile;
901
902 if (!options::cs_profile_path.empty())
903 Conf.CSIRProfile = options::cs_profile_path;
904 Conf.RunCSIRInstr = options::cs_pgo_gen;
905893
906894 Conf.DwoDir = options::dwo_dir;
907895
102102 static cl::opt
103103 SamplePGOFile("lto-sample-profile-file",
104104 cl::desc("Specify a SamplePGO profile file"));
105
106 static cl::opt
107 CSPGOFile("lto-cspgo-profile-file",
108 cl::desc("Specify a context sensitive PGO profile file"));
109
110 static cl::opt
111 RunCSIRInstr("lto-cspgo-gen",
112 cl::desc("Run PGO context sensitive IR instrumentation"),
113 cl::init(false), cl::Hidden);
114105
115106 static cl::opt
116107 UseNewPM("use-new-pm",
222213 Conf.RemarksWithHotness = OptRemarksWithHotness;
223214
224215 Conf.SampleProfile = SamplePGOFile;
225 Conf.CSIRProfile = CSPGOFile;
226 Conf.RunCSIRInstr = RunCSIRInstr;
227216
228217 // Run a custom pipeline, if asked for.
229218 Conf.OptPipeline = OptPipeline;