llvm.org GIT mirror llvm / df22060
IR: Use pointers instead of GUIDs to represent edges in the module summary. NFCI. When profiling a no-op incremental link of Chromium I found that the functions computeImportForFunction and computeDeadSymbols were consuming roughly 10% of the profile. The goal of this change is to improve the performance of those functions by changing the map lookups that they were previously doing into pointer dereferences. This is achieved by changing the ValueInfo data structure to be a pointer to an element of the global value map owned by ModuleSummaryIndex, and changing reference lists in the GlobalValueSummary to hold ValueInfos instead of GUIDs. This means that a ValueInfo will take a client directly to the summary list for a given GUID. Differential Revision: https://reviews.llvm.org/D32471 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@302108 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 3 years ago
13 changed file(s) with 219 addition(s) and 232 deletion(s). Raw diff Collapse all Expand all
4444 }
4545 };
4646
47 /// Struct to hold value either by GUID or GlobalValue*. Values in combined
48 /// indexes as well as indirect calls are GUIDs, all others are GlobalValues.
47 class GlobalValueSummary;
48
49 typedef std::vector> GlobalValueSummaryList;
50
51 struct GlobalValueSummaryInfo {
52 /// The GlobalValue corresponding to this summary. This is only used in
53 /// per-module summaries.
54 const GlobalValue *GV = nullptr;
55
56 /// List of global value summary structures for a particular value held
57 /// in the GlobalValueMap. Requires a vector in the case of multiple
58 /// COMDAT values of the same name.
59 GlobalValueSummaryList SummaryList;
60 };
61
62 /// Map from global value GUID to corresponding summary structures. Use a
63 /// std::map rather than a DenseMap so that pointers to the map's value_type
64 /// (which are used by ValueInfo) are not invalidated by insertion. Also it will
65 /// likely incur less overhead, as the value type is not very small and the size
66 /// of the map is unknown, resulting in inefficiencies due to repeated
67 /// insertions and resizing.
68 typedef std::map
69 GlobalValueSummaryMapTy;
70
71 /// Struct that holds a reference to a particular GUID in a global value
72 /// summary.
4973 struct ValueInfo {
50 /// The value representation used in this instance.
51 enum ValueInfoKind {
52 VI_GUID,
53 VI_Value,
54 };
55
56 /// Union of the two possible value types.
57 union ValueUnion {
58 GlobalValue::GUID Id;
59 const GlobalValue *GV;
60 ValueUnion(GlobalValue::GUID Id) : Id(Id) {}
61 ValueUnion(const GlobalValue *GV) : GV(GV) {}
62 };
63
64 /// The value being represented.
65 ValueUnion TheValue;
66 /// The value representation.
67 ValueInfoKind Kind;
68 /// Constructor for a GUID value
69 ValueInfo(GlobalValue::GUID Id = 0) : TheValue(Id), Kind(VI_GUID) {}
70 /// Constructor for a GlobalValue* value
71 ValueInfo(const GlobalValue *V) : TheValue(V), Kind(VI_Value) {}
72 /// Accessor for GUID value
73 GlobalValue::GUID getGUID() const {
74 assert(Kind == VI_GUID && "Not a GUID type");
75 return TheValue.Id;
76 }
77 /// Accessor for GlobalValue* value
78 const GlobalValue *getValue() const {
79 assert(Kind == VI_Value && "Not a Value type");
80 return TheValue.GV;
81 }
82 bool isGUID() const { return Kind == VI_GUID; }
74 const GlobalValueSummaryMapTy::value_type *Ref = nullptr;
75 ValueInfo() = default;
76 ValueInfo(const GlobalValueSummaryMapTy::value_type *Ref) : Ref(Ref) {}
77 operator bool() const { return Ref; }
78
79 GlobalValue::GUID getGUID() const { return Ref->first; }
80 const GlobalValue *getValue() const { return Ref->second.GV; }
81 ArrayRef> getSummaryList() const {
82 return Ref->second.SummaryList;
83 }
8384 };
8485
8586 template <> struct DenseMapInfo {
86 static inline ValueInfo getEmptyKey() { return ValueInfo((GlobalValue *)-1); }
87 static inline ValueInfo getEmptyKey() {
88 return ValueInfo((GlobalValueSummaryMapTy::value_type *)-1);
89 }
8790 static inline ValueInfo getTombstoneKey() {
88 return ValueInfo((GlobalValue *)-2);
89 }
90 static bool isEqual(ValueInfo L, ValueInfo R) {
91 if (L.isGUID() != R.isGUID())
92 return false;
93 return L.isGUID() ? (L.getGUID() == R.getGUID())
94 : (L.getValue() == R.getValue());
95 }
96 static unsigned getHashValue(ValueInfo I) {
97 return I.isGUID() ? I.getGUID() : (uintptr_t)I.getValue();
98 }
91 return ValueInfo((GlobalValueSummaryMapTy::value_type *)-2);
92 }
93 static bool isEqual(ValueInfo L, ValueInfo R) { return L.Ref == R.Ref; }
94 static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.Ref; }
9995 };
10096
10197 /// \brief Function and variable summary information to aid decisions and
482478 /// 160 bits SHA1
483479 typedef std::array ModuleHash;
484480
485 /// List of global value summary structures for a particular value held
486 /// in the GlobalValueMap. Requires a vector in the case of multiple
487 /// COMDAT values of the same name.
488 typedef std::vector> GlobalValueSummaryList;
489
490 /// Map from global value GUID to corresponding summary structures.
491 /// Use a std::map rather than a DenseMap since it will likely incur
492 /// less overhead, as the value type is not very small and the size
493 /// of the map is unknown, resulting in inefficiencies due to repeated
494 /// insertions and resizing.
495 typedef std::map
496 GlobalValueSummaryMapTy;
497
498481 /// Type used for iterating through the global value summary map.
499482 typedef GlobalValueSummaryMapTy::const_iterator const_gvsummary_iterator;
500483 typedef GlobalValueSummaryMapTy::iterator gvsummary_iterator;
531514 // YAML I/O support.
532515 friend yaml::MappingTraits;
533516
517 GlobalValueSummaryMapTy::value_type *
518 getOrInsertValuePtr(GlobalValue::GUID GUID) {
519 return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo{}).first;
520 }
521
534522 public:
535523 gvsummary_iterator begin() { return GlobalValueMap.begin(); }
536524 const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); }
538526 const_gvsummary_iterator end() const { return GlobalValueMap.end(); }
539527 size_t size() const { return GlobalValueMap.size(); }
540528
541 /// Get the list of global value summary objects for a given value name.
542 const GlobalValueSummaryList &getGlobalValueSummaryList(StringRef ValueName) {
543 return GlobalValueMap[GlobalValue::getGUID(ValueName)];
544 }
545
546 /// Get the list of global value summary objects for a given value name.
547 const const_gvsummary_iterator
548 findGlobalValueSummaryList(StringRef ValueName) const {
549 return GlobalValueMap.find(GlobalValue::getGUID(ValueName));
550 }
551
552 /// Get the list of global value summary objects for a given value GUID.
553 const const_gvsummary_iterator
554 findGlobalValueSummaryList(GlobalValue::GUID ValueGUID) const {
555 return GlobalValueMap.find(ValueGUID);
529 /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo().
530 ValueInfo getValueInfo(GlobalValue::GUID GUID) const {
531 auto I = GlobalValueMap.find(GUID);
532 return ValueInfo(I == GlobalValueMap.end() ? nullptr : &*I);
533 }
534
535 /// Return a ValueInfo for \p GUID.
536 ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID) {
537 return ValueInfo(getOrInsertValuePtr(GUID));
538 }
539
540 /// Return a ValueInfo for \p GV and mark it as belonging to GV.
541 ValueInfo getOrInsertValueInfo(const GlobalValue *GV) {
542 auto VP = getOrInsertValuePtr(GV->getGUID());
543 VP->second.GV = GV;
544 return ValueInfo(VP);
556545 }
557546
558547 /// Return the GUID for \p OriginalId in the OidGuidMap.
564553 /// Add a global value summary for a value of the given name.
565554 void addGlobalValueSummary(StringRef ValueName,
566555 std::unique_ptr Summary) {
567 addOriginalName(GlobalValue::getGUID(ValueName),
568 Summary->getOriginalName());
569 GlobalValueMap[GlobalValue::getGUID(ValueName)].push_back(
570 std::move(Summary));
571 }
572
573 /// Add a global value summary for a value of the given GUID.
574 void addGlobalValueSummary(GlobalValue::GUID ValueGUID,
556 addGlobalValueSummary(getOrInsertValueInfo(GlobalValue::getGUID(ValueName)),
557 std::move(Summary));
558 }
559
560 /// Add a global value summary for the given ValueInfo.
561 void addGlobalValueSummary(ValueInfo VI,
575562 std::unique_ptr Summary) {
576 addOriginalName(ValueGUID, Summary->getOriginalName());
577 GlobalValueMap[ValueGUID].push_back(std::move(Summary));
563 addOriginalName(VI.getGUID(), Summary->getOriginalName());
564 // Here we have a notionally const VI, but the value it points to is owned
565 // by the non-const *this.
566 const_cast(VI.Ref)
567 ->second.SummaryList.push_back(std::move(Summary));
578568 }
579569
580570 /// Add an original name for the value of the given GUID.
592582 /// not found.
593583 GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID,
594584 StringRef ModuleId) const {
595 auto CalleeInfoList = findGlobalValueSummaryList(ValueGUID);
596 if (CalleeInfoList == end()) {
585 auto CalleeInfo = getValueInfo(ValueGUID);
586 if (!CalleeInfo) {
597587 return nullptr; // This function does not have a summary
598588 }
599589 auto Summary =
600 llvm::find_if(CalleeInfoList->second,
590 llvm::find_if(CalleeInfo.getSummaryList(),
601591 [&](const std::unique_ptr &Summary) {
602592 return Summary->modulePath() == ModuleId;
603593 });
604 if (Summary == CalleeInfoList->second.end())
594 if (Summary == CalleeInfo.getSummaryList().end())
605595 return nullptr;
606596 return Summary->get();
607597 }
200200 for (auto &FSum : FSums) {
201201 GlobalValueSummary::GVFlags GVFlags(GlobalValue::ExternalLinkage, false,
202202 false);
203 Elem.push_back(llvm::make_unique(
203 Elem.SummaryList.push_back(llvm::make_unique(
204204 GVFlags, 0, ArrayRef{},
205205 ArrayRef{}, std::move(FSum.TypeTests),
206206 std::move(FSum.TypeTestAssumeVCalls),
212212 static void output(IO &io, GlobalValueSummaryMapTy &V) {
213213 for (auto &P : V) {
214214 std::vector FSums;
215 for (auto &Sum : P.second) {
215 for (auto &Sum : P.second.SummaryList) {
216216 if (auto *FSum = dyn_cast(Sum.get()))
217217 FSums.push_back(FunctionSummaryYaml{
218218 FSum->type_tests(), FSum->type_test_assume_vcalls(),
3636 // Walk through the operands of a given User via worklist iteration and populate
3737 // the set of GlobalValue references encountered. Invoked either on an
3838 // Instruction or a GlobalVariable (which walks its initializer).
39 static void findRefEdges(const User *CurUser, SetVector &RefEdges,
39 static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
40 SetVector &RefEdges,
4041 SmallPtrSet &Visited) {
4142 SmallVector Worklist;
4243 Worklist.push_back(CurUser);
6061 // the reference set unless it is a callee. Callees are handled
6162 // specially by WriteFunction and are added to a separate list.
6263 if (!(CS && CS.isCallee(&OI)))
63 RefEdges.insert(GV);
64 RefEdges.insert(Index.getOrInsertValueInfo(GV));
6465 continue;
6566 }
6667 Worklist.push_back(Operand);
197198 if (isa(I))
198199 continue;
199200 ++NumInsts;
200 findRefEdges(&I, RefEdges, Visited);
201 findRefEdges(Index, &I, RefEdges, Visited);
201202 auto CS = ImmutableCallSite(&I);
202203 if (!CS)
203204 continue;
238239 // to record the call edge to the alias in that case. Eventually
239240 // an alias summary will be created to associate the alias and
240241 // aliasee.
241 CallGraphEdges[cast(CalledValue)].updateHotness(Hotness);
242 CallGraphEdges[Index.getOrInsertValueInfo(
243 cast(CalledValue))]
244 .updateHotness(Hotness);
242245 } else {
243246 // Skip inline assembly calls.
244247 if (CI && CI->isInlineAsm())
253256 ICallAnalysis.getPromotionCandidatesForInstruction(
254257 &I, NumVals, TotalCount, NumCandidates);
255258 for (auto &Candidate : CandidateProfileData)
256 CallGraphEdges[Candidate.Value].updateHotness(
257 getHotness(Candidate.Count, PSI));
259 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
260 .updateHotness(getHotness(Candidate.Count, PSI));
258261 }
259262 }
260263
261264 // Explicit add hot edges to enforce importing for designated GUIDs for
262265 // sample PGO, to enable the same inlines as the profiled optimized binary.
263266 for (auto &I : F.getImportGUIDs())
264 CallGraphEdges[I].updateHotness(CalleeInfo::HotnessType::Hot);
267 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
268 CalleeInfo::HotnessType::Hot);
265269
266270 bool NonRenamableLocal = isNonRenamableLocal(F);
267271 bool NotEligibleForImport =
287291 DenseSet &CantBePromoted) {
288292 SetVector RefEdges;
289293 SmallPtrSet Visited;
290 findRefEdges(&V, RefEdges, Visited);
294 findRefEdges(Index, &V, RefEdges, Visited);
291295 bool NonRenamableLocal = isNonRenamableLocal(V);
292296 GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
293297 /* LiveRoot = */ false);
316320
317321 // Set LiveRoot flag on entries matching the given value name.
318322 static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
319 auto SummaryList =
320 Index.findGlobalValueSummaryList(GlobalValue::getGUID(Name));
321 if (SummaryList == Index.end())
322 return;
323 for (auto &Summary : SummaryList->second)
324 Summary->setLiveRoot();
323 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
324 for (auto &Summary : VI.getSummaryList())
325 Summary->setLiveRoot();
325326 }
326327
327328 ModuleSummaryIndex llvm::buildModuleSummaryIndex(
445446 }
446447
447448 for (auto &GlobalList : Index) {
448 assert(GlobalList.second.size() == 1 &&
449 // Ignore entries for references that are undefined in the current module.
450 if (GlobalList.second.SummaryList.empty())
451 continue;
452
453 assert(GlobalList.second.SummaryList.size() == 1 &&
449454 "Expected module's index to have one summary per GUID");
450 auto &Summary = GlobalList.second[0];
455 auto &Summary = GlobalList.second.SummaryList[0];
451456 bool AllRefsCanBeExternallyReferenced =
452457 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
453 return !CantBePromoted.count(VI.getValue()->getGUID());
458 return !CantBePromoted.count(VI.getGUID());
454459 });
455460 if (!AllRefsCanBeExternallyReferenced) {
456461 Summary->setNotEligibleToImport();
460465 if (auto *FuncSummary = dyn_cast(Summary.get())) {
461466 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
462467 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
463 auto GUID = Edge.first.isGUID() ? Edge.first.getGUID()
464 : Edge.first.getValue()->getGUID();
465 return !CantBePromoted.count(GUID);
468 return !CantBePromoted.count(Edge.first.getGUID());
466469 });
467470 if (!AllCallsCanBeExternallyReferenced)
468471 Summary->setNotEligibleToImport();
693693 /// Used to enable on-demand parsing of the VST.
694694 uint64_t VSTOffset = 0;
695695
696 // Map to save ValueId to GUID association that was recorded in the
696 // Map to save ValueId to ValueInfo association that was recorded in the
697697 // ValueSymbolTable. It is used after the VST is parsed to convert
698698 // call graph edges read from the function summary from referencing
699 // callees by their ValueId to using the GUID instead, which is how
699 // callees by their ValueId to using the ValueInfo instead, which is how
700700 // they are recorded in the summary index being built.
701 // We save a second GUID which is the same as the first one, but ignoring the
702 // linkage, i.e. for value other than local linkage they are identical.
703 DenseMap>
704 ValueIdToCallGraphGUIDMap;
701 // We save a GUID which refers to the same global as the ValueInfo, but
702 // ignoring the linkage, i.e. for values other than local linkage they are
703 // identical.
704 DenseMap>
705 ValueIdToValueInfoMap;
705706
706707 /// Map populated during module path string table parsing, from the
707708 /// module ID to a string reference owned by the index's module
741742 Error parseEntireSummary();
742743 Error parseModuleStringTable();
743744
744 std::pair
745 getGUIDFromValueId(unsigned ValueId);
745 std::pair
746 getValueInfoFromValueId(unsigned ValueId);
746747
747748 ModulePathStringTableTy::iterator addThisModulePath();
748749 };
46964697 return TheIndex.addModulePath(ModulePath, ModuleId);
46974698 }
46984699
4699 std::pair
4700 ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) {
4701 auto VGI = ValueIdToCallGraphGUIDMap.find(ValueId);
4702 assert(VGI != ValueIdToCallGraphGUIDMap.end());
4703 return VGI->second;
4700 std::pair
4701 ModuleSummaryIndexBitcodeReader::getValueInfoFromValueId(unsigned ValueId) {
4702 auto VGI = ValueIdToValueInfoMap[ValueId];
4703 assert(VGI.first);
4704 return VGI;
47044705 }
47054706
47064707 void ModuleSummaryIndexBitcodeReader::setValueGUID(
47154716 if (PrintSummaryGUIDs)
47164717 dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is "
47174718 << ValueName << "\n";
4718 ValueIdToCallGraphGUIDMap[ValueID] =
4719 std::make_pair(ValueGUID, OriginalNameID);
4719 ValueIdToValueInfoMap[ValueID] =
4720 std::make_pair(TheIndex.getOrInsertValueInfo(ValueGUID), OriginalNameID);
47204721 }
47214722
47224723 // Specialized value symbol table parser used when reading module index
47944795 GlobalValue::GUID RefGUID = Record[1];
47954796 // The "original name", which is the second value of the pair will be
47964797 // overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index.
4797 ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID);
4798 ValueIdToValueInfoMap[ValueID] =
4799 std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
47984800 break;
47994801 }
48004802 }
49394941 std::vector Ret;
49404942 Ret.reserve(Record.size());
49414943 for (uint64_t RefValueId : Record)
4942 Ret.push_back(getGUIDFromValueId(RefValueId).first);
4944 Ret.push_back(getValueInfoFromValueId(RefValueId).first);
49434945 return Ret;
49444946 }
49454947
49494951 Ret.reserve(Record.size());
49504952 for (unsigned I = 0, E = Record.size(); I != E; ++I) {
49514953 CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown;
4952 GlobalValue::GUID CalleeGUID = getGUIDFromValueId(Record[I]).first;
4954 ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
49534955 if (IsOldProfileFormat) {
49544956 I += 1; // Skip old callsitecount field
49554957 if (HasProfile)
49564958 I += 1; // Skip old profilecount field
49574959 } else if (HasProfile)
49584960 Hotness = static_cast(Record[++I]);
4959 Ret.push_back(FunctionSummary::EdgeTy{CalleeGUID, CalleeInfo{Hotness}});
4961 Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}});
49604962 }
49614963 return Ret;
49624964 }
50265028 case bitc::FS_VALUE_GUID: { // [valueid, refguid]
50275029 uint64_t ValueID = Record[0];
50285030 GlobalValue::GUID RefGUID = Record[1];
5029 ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID);
5031 ValueIdToValueInfoMap[ValueID] =
5032 std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
50305033 break;
50315034 }
50325035 // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid,
50675070 PendingTypeCheckedLoadVCalls.clear();
50685071 PendingTypeTestAssumeConstVCalls.clear();
50695072 PendingTypeCheckedLoadConstVCalls.clear();
5070 auto GUID = getGUIDFromValueId(ValueID);
5073 auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID);
50715074 FS->setModulePath(addThisModulePath()->first());
5072 FS->setOriginalName(GUID.second);
5073 TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
5075 FS->setOriginalName(VIAndOriginalGUID.second);
5076 TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS));
50745077 break;
50755078 }
50765079 // FS_ALIAS: [valueid, flags, valueid]
50905093 // ownership.
50915094 AS->setModulePath(addThisModulePath()->first());
50925095
5093 GlobalValue::GUID AliaseeGUID = getGUIDFromValueId(AliaseeID).first;
5096 GlobalValue::GUID AliaseeGUID =
5097 getValueInfoFromValueId(AliaseeID).first.getGUID();
50945098 auto AliaseeInModule =
50955099 TheIndex.findSummaryInModule(AliaseeGUID, ModulePath);
50965100 if (!AliaseeInModule)
50975101 return error("Alias expects aliasee summary to be parsed");
50985102 AS->setAliasee(AliaseeInModule);
50995103
5100 auto GUID = getGUIDFromValueId(ValueID);
5104 auto GUID = getValueInfoFromValueId(ValueID);
51015105 AS->setOriginalName(GUID.second);
51025106 TheIndex.addGlobalValueSummary(GUID.first, std::move(AS));
51035107 break;
51115115 makeRefList(ArrayRef(Record).slice(2));
51125116 auto FS = llvm::make_unique(Flags, std::move(Refs));
51135117 FS->setModulePath(addThisModulePath()->first());
5114 auto GUID = getGUIDFromValueId(ValueID);
5118 auto GUID = getValueInfoFromValueId(ValueID);
51155119 FS->setOriginalName(GUID.second);
51165120 TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
51175121 break;
51385142 std::vector Edges = makeCallList(
51395143 ArrayRef(Record).slice(CallGraphEdgeStartIndex),
51405144 IsOldProfileFormat, HasProfile);
5141 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5145 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
51425146 auto FS = llvm::make_unique(
51435147 Flags, InstCount, std::move(Refs), std::move(Edges),
51445148 std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls),
51515155 PendingTypeTestAssumeConstVCalls.clear();
51525156 PendingTypeCheckedLoadConstVCalls.clear();
51535157 LastSeenSummary = FS.get();
5154 LastSeenGUID = GUID;
5158 LastSeenGUID = VI.getGUID();
51555159 FS->setModulePath(ModuleIdMap[ModuleId]);
5156 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
5160 TheIndex.addGlobalValueSummary(VI, std::move(FS));
51575161 break;
51585162 }
51595163 // FS_COMBINED_ALIAS: [valueid, modid, flags, valueid]
51695173 LastSeenSummary = AS.get();
51705174 AS->setModulePath(ModuleIdMap[ModuleId]);
51715175
5172 auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first;
5176 auto AliaseeGUID =
5177 getValueInfoFromValueId(AliaseeValueId).first.getGUID();
51735178 auto AliaseeInModule =
51745179 TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath());
51755180 if (!AliaseeInModule)
51765181 return error("Alias expects aliasee summary to be parsed");
51775182 AS->setAliasee(AliaseeInModule);
51785183
5179 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5180 LastSeenGUID = GUID;
5181 TheIndex.addGlobalValueSummary(GUID, std::move(AS));
5184 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
5185 LastSeenGUID = VI.getGUID();
5186 TheIndex.addGlobalValueSummary(VI, std::move(AS));
51825187 break;
51835188 }
51845189 // FS_COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid]
51925197 auto FS = llvm::make_unique(Flags, std::move(Refs));
51935198 LastSeenSummary = FS.get();
51945199 FS->setModulePath(ModuleIdMap[ModuleId]);
5195 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5196 LastSeenGUID = GUID;
5197 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
5200 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
5201 LastSeenGUID = VI.getGUID();
5202 TheIndex.addGlobalValueSummary(VI, std::move(FS));
51985203 break;
51995204 }
52005205 // FS_COMBINED_ORIGINAL_NAME: [original_name]
155155 return;
156156 for (const auto &GUIDSummaryLists : *Index)
157157 // Examine all summaries for this GUID.
158 for (auto &Summary : GUIDSummaryLists.second)
158 for (auto &Summary : GUIDSummaryLists.second.SummaryList)
159159 if (auto FS = dyn_cast(Summary.get()))
160160 // For each call in the function summary, see if the call
161161 // is to a GUID (which means it is for an indirect call,
162162 // otherwise we would have a Value for it). If so, synthesize
163163 // a value id.
164164 for (auto &CallEdge : FS->calls())
165 if (CallEdge.first.isGUID())
165 if (!CallEdge.first.getValue())
166166 assignValueId(CallEdge.first.getGUID());
167167 }
168168
303303 }
304304 // Helper to get the valueId for the type of value recorded in VI.
305305 unsigned getValueId(ValueInfo VI) {
306 if (VI.isGUID())
306 if (!VI.getValue())
307307 return getValueId(VI.getGUID());
308308 return VE.getValueID(VI.getValue());
309309 }
357357 Callback(Summary);
358358 } else {
359359 for (auto &Summaries : Index)
360 for (auto &Summary : Summaries.second)
360 for (auto &Summary : Summaries.second.SummaryList)
361361 Callback({Summaries.first, Summary.get()});
362362 }
363363 }
32693269 void ModuleBitcodeWriter::writeModuleLevelReferences(
32703270 const GlobalVariable &V, SmallVector &NameVals,
32713271 unsigned FSModRefsAbbrev) {
3272 auto Summaries =
3273 Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName()));
3274 if (Summaries == Index->end()) {
3272 auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName()));
3273 if (!VI || VI.getSummaryList().empty()) {
32753274 // Only declarations should not have a summary (a declaration might however
32763275 // have a summary if the def was in module level asm).
32773276 assert(V.isDeclaration());
32783277 return;
32793278 }
3280 auto *Summary = Summaries->second.front().get();
3279 auto *Summary = VI.getSummaryList()[0].get();
32813280 NameVals.push_back(VE.getValueID(&V));
32823281 GlobalVarSummary *VS = cast(Summary);
32833282 NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
33663365 if (!F.hasName())
33673366 report_fatal_error("Unexpected anonymous function when writing summary");
33683367
3369 auto Summaries =
3370 Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName()));
3371 if (Summaries == Index->end()) {
3368 ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName()));
3369 if (!VI || VI.getSummaryList().empty()) {
33723370 // Only declarations should not have a summary (a declaration might
33733371 // however have a summary if the def was in module level asm).
33743372 assert(F.isDeclaration());
33753373 continue;
33763374 }
3377 auto *Summary = Summaries->second.front().get();
3375 auto *Summary = VI.getSummaryList()[0].get();
33783376 writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F),
33793377 FSCallsAbbrev, FSCallsProfileAbbrev, F);
33803378 }
2121 StringRef ModulePath, GVSummaryMapTy &GVSummaryMap) const {
2222 for (auto &GlobalList : *this) {
2323 auto GUID = GlobalList.first;
24 for (auto &GlobSummary : GlobalList.second) {
24 for (auto &GlobSummary : GlobalList.second.SummaryList) {
2525 auto *Summary = dyn_cast_or_null(GlobSummary.get());
2626 if (!Summary)
2727 // Ignore global variable, focus on functions
3939 StringMap &ModuleToDefinedGVSummaries) const {
4040 for (auto &GlobalList : *this) {
4141 auto GUID = GlobalList.first;
42 for (auto &Summary : GlobalList.second) {
42 for (auto &Summary : GlobalList.second.SummaryList) {
4343 ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get();
4444 }
4545 }
4848 GlobalValueSummary *
4949 ModuleSummaryIndex::getGlobalValueSummary(uint64_t ValueGUID,
5050 bool PerModuleIndex) const {
51 auto SummaryList = findGlobalValueSummaryList(ValueGUID);
52 assert(SummaryList != end() && "GlobalValue not found in index");
53 assert((!PerModuleIndex || SummaryList->second.size() == 1) &&
51 auto VI = getValueInfo(ValueGUID);
52 assert(VI && "GlobalValue not found in index");
53 assert((!PerModuleIndex || VI.getSummaryList().size() == 1) &&
5454 "Expected a single entry per global value in per-module index");
55 auto &Summary = SummaryList->second[0];
55 auto &Summary = VI.getSummaryList()[0];
5656 return Summary.get();
5757 }
273273 // when needed.
274274 DenseSet GlobalInvolvedWithAlias;
275275 for (auto &I : Index)
276 for (auto &S : I.second)
276 for (auto &S : I.second.SummaryList)
277277 if (auto AS = dyn_cast(S.get()))
278278 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
279279
280280 for (auto &I : Index)
281 thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias,
282 isPrevailing, recordNewLinkage);
281 thinLTOResolveWeakForLinkerGUID(I.second.SummaryList, I.first,
282 GlobalInvolvedWithAlias, isPrevailing,
283 recordNewLinkage);
283284 }
284285
285286 static void thinLTOInternalizeAndPromoteGUID(
300301 ModuleSummaryIndex &Index,
301302 function_ref isExported) {
302303 for (auto &I : Index)
303 thinLTOInternalizeAndPromoteGUID(I.second, I.first, isExported);
304 thinLTOInternalizeAndPromoteGUID(I.second.SummaryList, I.first, isExported);
304305 }
305306
306307 // Requires a destructor for std::vector.
118118 };
119119
120120 for (auto &I : Index) {
121 if (HasMultipleCopies(I.second))
122 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second);
121 if (HasMultipleCopies(I.second.SummaryList))
122 PrevailingCopy[I.first] =
123 getFirstDefinitionForLinker(I.second.SummaryList);
123124 }
124125 }
125126
116116 /// - [insert you fancy metric here]
117117 static const GlobalValueSummary *
118118 selectCallee(const ModuleSummaryIndex &Index,
119 const GlobalValueSummaryList &CalleeSummaryList,
119 ArrayRef> CalleeSummaryList,
120120 unsigned Threshold, StringRef CallerModulePath) {
121121 auto It = llvm::find_if(
122122 CalleeSummaryList,
167167 return cast(It->get());
168168 }
169169
170 /// Return the summary for the function \p GUID that fits the \p Threshold, or
171 /// null if there's no match.
172 static const GlobalValueSummary *selectCallee(GlobalValue::GUID GUID,
173 unsigned Threshold,
174 const ModuleSummaryIndex &Index,
175 StringRef CallerModulePath) {
176 auto CalleeSummaryList = Index.findGlobalValueSummaryList(GUID);
177 if (CalleeSummaryList == Index.end())
178 return nullptr; // This function does not have a summary
179 return selectCallee(Index, CalleeSummaryList->second, Threshold,
180 CallerModulePath);
181 }
182
183170 using EdgeInfo = std::tuple
184171 GlobalValue::GUID>;
185172
193180 FunctionImporter::ImportMapTy &ImportList,
194181 StringMap *ExportLists = nullptr) {
195182 for (auto &Edge : Summary.calls()) {
196 auto GUID = Edge.first.getGUID();
197 DEBUG(dbgs() << " edge -> " << GUID << " Threshold:" << Threshold << "\n");
198
199 if (Index.findGlobalValueSummaryList(GUID) == Index.end()) {
183 ValueInfo VI = Edge.first;
184 DEBUG(dbgs() << " edge -> " << VI.getGUID() << " Threshold:" << Threshold
185 << "\n");
186
187 if (VI.getSummaryList().empty()) {
200188 // For SamplePGO, the indirect call targets for local functions will
201189 // have its original name annotated in profile. We try to find the
202190 // corresponding PGOFuncName as the GUID.
203 GUID = Index.getGUIDFromOriginalID(GUID);
191 auto GUID = Index.getGUIDFromOriginalID(VI.getGUID());
204192 if (GUID == 0)
205193 continue;
206 }
207
208 if (DefinedGVSummaries.count(GUID)) {
194 VI = Index.getValueInfo(GUID);
195 if (!VI)
196 continue;
197 }
198
199 if (DefinedGVSummaries.count(VI.getGUID())) {
209200 DEBUG(dbgs() << "ignored! Target already in destination module.\n");
210201 continue;
211202 }
221212 const auto NewThreshold =
222213 Threshold * GetBonusMultiplier(Edge.second.Hotness);
223214
224 auto *CalleeSummary =
225 selectCallee(GUID, NewThreshold, Index, Summary.modulePath());
215 auto *CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold,
216 Summary.modulePath());
226217 if (!CalleeSummary) {
227218 DEBUG(dbgs() << "ignored! No qualifying callee with summary found.\n");
228219 continue;
254245 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
255246
256247 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
257 auto &ProcessedThreshold = ImportList[ExportModulePath][GUID];
248 auto &ProcessedThreshold = ImportList[ExportModulePath][VI.getGUID()];
258249 /// Since the traversal of the call graph is DFS, we can revisit a function
259250 /// a second time with a higher threshold. In this case, it is added back to
260251 /// the worklist with the new threshold.
270261 // Make exports in the source module.
271262 if (ExportLists) {
272263 auto &ExportList = (*ExportLists)[ExportModulePath];
273 ExportList.insert(GUID);
264 ExportList.insert(VI.getGUID());
274265 if (!PreviouslyImported) {
275266 // This is the first time this function was exported from its source
276267 // module, so mark all functions and globals it references as exported
290281 }
291282
292283 // Insert the newly imported function to the worklist.
293 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, GUID);
284 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID());
294285 }
295286 }
296287
430421 if (GUIDPreservedSymbols.empty())
431422 // Don't do anything when nothing is live, this is friendly with tests.
432423 return DenseSet();
433 DenseSet LiveSymbols = GUIDPreservedSymbols;
434 SmallVector Worklist;
435 Worklist.reserve(LiveSymbols.size() * 2);
436 for (auto GUID : LiveSymbols) {
437 DEBUG(dbgs() << "Live root: " << GUID << "\n");
438 Worklist.push_back(GUID);
424 DenseSet LiveSymbols;
425 SmallVector Worklist;
426 Worklist.reserve(GUIDPreservedSymbols.size() * 2);
427 for (auto GUID : GUIDPreservedSymbols) {
428 ValueInfo VI = Index.getValueInfo(GUID);
429 if (!VI)
430 continue;
431 DEBUG(dbgs() << "Live root: " << VI.getGUID() << "\n");
432 LiveSymbols.insert(VI);
433 Worklist.push_back(VI);
439434 }
440435 // Add values flagged in the index as live roots to the worklist.
441436 for (const auto &Entry : Index) {
442437 bool IsLiveRoot = llvm::any_of(
443 Entry.second,
438 Entry.second.SummaryList,
444439 [&](const std::unique_ptr &Summary) {
445440 return Summary->liveRoot();
446441 });
447442 if (!IsLiveRoot)
448443 continue;
449444 DEBUG(dbgs() << "Live root (summary): " << Entry.first << "\n");
450 Worklist.push_back(Entry.first);
445 Worklist.push_back(ValueInfo(&Entry));
451446 }
452447
453448 while (!Worklist.empty()) {
454 auto GUID = Worklist.pop_back_val();
455 auto It = Index.findGlobalValueSummaryList(GUID);
456 if (It == Index.end()) {
457 DEBUG(dbgs() << "Not in index: " << GUID << "\n");
458 continue;
459 }
449 auto VI = Worklist.pop_back_val();
460450
461451 // FIXME: we should only make the prevailing copy live here
462 for (auto &Summary : It->second) {
452 for (auto &Summary : VI.getSummaryList()) {
463453 for (auto Ref : Summary->refs()) {
464 auto RefGUID = Ref.getGUID();
465 if (LiveSymbols.insert(RefGUID).second) {
466 DEBUG(dbgs() << "Marking live (ref): " << RefGUID << "\n");
467 Worklist.push_back(RefGUID);
454 if (LiveSymbols.insert(Ref).second) {
455 DEBUG(dbgs() << "Marking live (ref): " << Ref.getGUID() << "\n");
456 Worklist.push_back(Ref);
468457 }
469458 }
470459 if (auto *FS = dyn_cast(Summary.get())) {
471460 for (auto Call : FS->calls()) {
472 auto CallGUID = Call.first.getGUID();
473 if (LiveSymbols.insert(CallGUID).second) {
474 DEBUG(dbgs() << "Marking live (call): " << CallGUID << "\n");
475 Worklist.push_back(CallGUID);
461 if (LiveSymbols.insert(Call.first).second) {
462 DEBUG(dbgs() << "Marking live (call): " << Call.first.getGUID()
463 << "\n");
464 Worklist.push_back(Call.first);
476465 }
477466 }
478467 }
479468 if (auto *AS = dyn_cast(Summary.get())) {
480469 auto AliaseeGUID = AS->getAliasee().getOriginalName();
481 if (LiveSymbols.insert(AliaseeGUID).second) {
470 ValueInfo AliaseeVI = Index.getValueInfo(AliaseeGUID);
471 if (AliaseeVI && LiveSymbols.insert(AliaseeVI).second) {
482472 DEBUG(dbgs() << "Marking live (alias): " << AliaseeGUID << "\n");
483 Worklist.push_back(AliaseeGUID);
473 Worklist.push_back(AliaseeVI);
484474 }
485475 }
486476 }
489479 DeadSymbols.reserve(
490480 std::min(Index.size(), Index.size() - LiveSymbols.size()));
491481 for (auto &Entry : Index) {
492 auto GUID = Entry.first;
493 if (!LiveSymbols.count(GUID)) {
494 DEBUG(dbgs() << "Marking dead: " << GUID << "\n");
495 DeadSymbols.insert(GUID);
482 if (!LiveSymbols.count(ValueInfo(&Entry))) {
483 DEBUG(dbgs() << "Marking dead: " << Entry.first << "\n");
484 DeadSymbols.insert(Entry.first);
496485 }
497486 }
498487 DEBUG(dbgs() << LiveSymbols.size() << " symbols Live, and "
824813 // is only enabled when testing importing via the 'opt' tool, which does
825814 // not do the ThinLink that would normally determine what values to promote.
826815 for (auto &I : *Index) {
827 for (auto &S : I.second) {
816 for (auto &S : I.second.SummaryList) {
828817 if (GlobalValue::isLocalLinkage(S->linkage()))
829818 S->setLinkage(GlobalValue::ExternalLinkage);
830819 }
14391439 }
14401440
14411441 for (auto &P : *ExportSummary) {
1442 for (auto &S : P.second) {
1442 for (auto &S : P.second.SummaryList) {
14431443 auto *FS = dyn_cast(S.get());
14441444 if (!FS)
14451445 continue;
13211321 }
13221322
13231323 for (auto &P : *ExportSummary) {
1324 for (auto &S : P.second) {
1324 for (auto &S : P.second.SummaryList) {
13251325 auto *FS = dyn_cast(S.get());
13261326 if (!FS)
13271327 continue;
299299 // does not do the ThinLink that would normally determine what values to
300300 // promote.
301301 for (auto &I : *Index) {
302 for (auto &S : I.second) {
302 for (auto &S : I.second.SummaryList) {
303303 if (GlobalValue::isLocalLinkage(S->linkage()))
304304 S->setLinkage(GlobalValue::ExternalLinkage);
305305 }
283283
284284 unsigned Calls = 0, Refs = 0, Functions = 0, Alias = 0, Globals = 0;
285285 for (auto &Summaries : *Index) {
286 for (auto &Summary : Summaries.second) {
286 for (auto &Summary : Summaries.second.SummaryList) {
287287 Refs += Summary->refs().size();
288288 if (auto *FuncSummary = dyn_cast(Summary.get())) {
289289 Functions++;