llvm.org GIT mirror llvm / e611018
Re-apply r302108, "IR: Use pointers instead of GUIDs to represent edges in the module summary. NFCI." with a fix for the clang backend. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@302176 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 2 years ago
13 changed file(s) with 219 addition(s) and 232 deletion(s). Raw diff Collapse all Expand all
4444 }
4545 };
4646
47 /// Struct to hold value either by GUID or GlobalValue*. Values in combined
48 /// indexes as well as indirect calls are GUIDs, all others are GlobalValues.
47 class GlobalValueSummary;
48
49 typedef std::vector> GlobalValueSummaryList;
50
51 struct GlobalValueSummaryInfo {
52 /// The GlobalValue corresponding to this summary. This is only used in
53 /// per-module summaries.
54 const GlobalValue *GV = nullptr;
55
56 /// List of global value summary structures for a particular value held
57 /// in the GlobalValueMap. Requires a vector in the case of multiple
58 /// COMDAT values of the same name.
59 GlobalValueSummaryList SummaryList;
60 };
61
62 /// Map from global value GUID to corresponding summary structures. Use a
63 /// std::map rather than a DenseMap so that pointers to the map's value_type
64 /// (which are used by ValueInfo) are not invalidated by insertion. Also it will
65 /// likely incur less overhead, as the value type is not very small and the size
66 /// of the map is unknown, resulting in inefficiencies due to repeated
67 /// insertions and resizing.
68 typedef std::map
69 GlobalValueSummaryMapTy;
70
71 /// Struct that holds a reference to a particular GUID in a global value
72 /// summary.
4973 struct ValueInfo {
50 /// The value representation used in this instance.
51 enum ValueInfoKind {
52 VI_GUID,
53 VI_Value,
54 };
55
56 /// Union of the two possible value types.
57 union ValueUnion {
58 GlobalValue::GUID Id;
59 const GlobalValue *GV;
60 ValueUnion(GlobalValue::GUID Id) : Id(Id) {}
61 ValueUnion(const GlobalValue *GV) : GV(GV) {}
62 };
63
64 /// The value being represented.
65 ValueUnion TheValue;
66 /// The value representation.
67 ValueInfoKind Kind;
68 /// Constructor for a GUID value
69 ValueInfo(GlobalValue::GUID Id = 0) : TheValue(Id), Kind(VI_GUID) {}
70 /// Constructor for a GlobalValue* value
71 ValueInfo(const GlobalValue *V) : TheValue(V), Kind(VI_Value) {}
72 /// Accessor for GUID value
73 GlobalValue::GUID getGUID() const {
74 assert(Kind == VI_GUID && "Not a GUID type");
75 return TheValue.Id;
76 }
77 /// Accessor for GlobalValue* value
78 const GlobalValue *getValue() const {
79 assert(Kind == VI_Value && "Not a Value type");
80 return TheValue.GV;
81 }
82 bool isGUID() const { return Kind == VI_GUID; }
74 const GlobalValueSummaryMapTy::value_type *Ref = nullptr;
75 ValueInfo() = default;
76 ValueInfo(const GlobalValueSummaryMapTy::value_type *Ref) : Ref(Ref) {}
77 operator bool() const { return Ref; }
78
79 GlobalValue::GUID getGUID() const { return Ref->first; }
80 const GlobalValue *getValue() const { return Ref->second.GV; }
81 ArrayRef> getSummaryList() const {
82 return Ref->second.SummaryList;
83 }
8384 };
8485
8586 template <> struct DenseMapInfo {
86 static inline ValueInfo getEmptyKey() { return ValueInfo((GlobalValue *)-1); }
87 static inline ValueInfo getEmptyKey() {
88 return ValueInfo((GlobalValueSummaryMapTy::value_type *)-1);
89 }
8790 static inline ValueInfo getTombstoneKey() {
88 return ValueInfo((GlobalValue *)-2);
89 }
90 static bool isEqual(ValueInfo L, ValueInfo R) {
91 if (L.isGUID() != R.isGUID())
92 return false;
93 return L.isGUID() ? (L.getGUID() == R.getGUID())
94 : (L.getValue() == R.getValue());
95 }
96 static unsigned getHashValue(ValueInfo I) {
97 return I.isGUID() ? I.getGUID() : (uintptr_t)I.getValue();
98 }
91 return ValueInfo((GlobalValueSummaryMapTy::value_type *)-2);
92 }
93 static bool isEqual(ValueInfo L, ValueInfo R) { return L.Ref == R.Ref; }
94 static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.Ref; }
9995 };
10096
10197 /// \brief Function and variable summary information to aid decisions and
482478 /// 160 bits SHA1
483479 typedef std::array ModuleHash;
484480
485 /// List of global value summary structures for a particular value held
486 /// in the GlobalValueMap. Requires a vector in the case of multiple
487 /// COMDAT values of the same name.
488 typedef std::vector> GlobalValueSummaryList;
489
490 /// Map from global value GUID to corresponding summary structures.
491 /// Use a std::map rather than a DenseMap since it will likely incur
492 /// less overhead, as the value type is not very small and the size
493 /// of the map is unknown, resulting in inefficiencies due to repeated
494 /// insertions and resizing.
495 typedef std::map
496 GlobalValueSummaryMapTy;
497
498481 /// Type used for iterating through the global value summary map.
499482 typedef GlobalValueSummaryMapTy::const_iterator const_gvsummary_iterator;
500483 typedef GlobalValueSummaryMapTy::iterator gvsummary_iterator;
531514 // YAML I/O support.
532515 friend yaml::MappingTraits;
533516
517 GlobalValueSummaryMapTy::value_type *
518 getOrInsertValuePtr(GlobalValue::GUID GUID) {
519 return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo{}).first;
520 }
521
534522 public:
535523 gvsummary_iterator begin() { return GlobalValueMap.begin(); }
536524 const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); }
538526 const_gvsummary_iterator end() const { return GlobalValueMap.end(); }
539527 size_t size() const { return GlobalValueMap.size(); }
540528
541 /// Get the list of global value summary objects for a given value name.
542 const GlobalValueSummaryList &getGlobalValueSummaryList(StringRef ValueName) {
543 return GlobalValueMap[GlobalValue::getGUID(ValueName)];
544 }
545
546 /// Get the list of global value summary objects for a given value name.
547 const const_gvsummary_iterator
548 findGlobalValueSummaryList(StringRef ValueName) const {
549 return GlobalValueMap.find(GlobalValue::getGUID(ValueName));
550 }
551
552 /// Get the list of global value summary objects for a given value GUID.
553 const const_gvsummary_iterator
554 findGlobalValueSummaryList(GlobalValue::GUID ValueGUID) const {
555 return GlobalValueMap.find(ValueGUID);
529 /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo().
530 ValueInfo getValueInfo(GlobalValue::GUID GUID) const {
531 auto I = GlobalValueMap.find(GUID);
532 return ValueInfo(I == GlobalValueMap.end() ? nullptr : &*I);
533 }
534
535 /// Return a ValueInfo for \p GUID.
536 ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID) {
537 return ValueInfo(getOrInsertValuePtr(GUID));
538 }
539
540 /// Return a ValueInfo for \p GV and mark it as belonging to GV.
541 ValueInfo getOrInsertValueInfo(const GlobalValue *GV) {
542 auto VP = getOrInsertValuePtr(GV->getGUID());
543 VP->second.GV = GV;
544 return ValueInfo(VP);
556545 }
557546
558547 /// Return the GUID for \p OriginalId in the OidGuidMap.
564553 /// Add a global value summary for a value of the given name.
565554 void addGlobalValueSummary(StringRef ValueName,
566555 std::unique_ptr Summary) {
567 addOriginalName(GlobalValue::getGUID(ValueName),
568 Summary->getOriginalName());
569 GlobalValueMap[GlobalValue::getGUID(ValueName)].push_back(
570 std::move(Summary));
571 }
572
573 /// Add a global value summary for a value of the given GUID.
574 void addGlobalValueSummary(GlobalValue::GUID ValueGUID,
556 addGlobalValueSummary(getOrInsertValueInfo(GlobalValue::getGUID(ValueName)),
557 std::move(Summary));
558 }
559
560 /// Add a global value summary for the given ValueInfo.
561 void addGlobalValueSummary(ValueInfo VI,
575562 std::unique_ptr Summary) {
576 addOriginalName(ValueGUID, Summary->getOriginalName());
577 GlobalValueMap[ValueGUID].push_back(std::move(Summary));
563 addOriginalName(VI.getGUID(), Summary->getOriginalName());
564 // Here we have a notionally const VI, but the value it points to is owned
565 // by the non-const *this.
566 const_cast(VI.Ref)
567 ->second.SummaryList.push_back(std::move(Summary));
578568 }
579569
580570 /// Add an original name for the value of the given GUID.
592582 /// not found.
593583 GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID,
594584 StringRef ModuleId) const {
595 auto CalleeInfoList = findGlobalValueSummaryList(ValueGUID);
596 if (CalleeInfoList == end()) {
585 auto CalleeInfo = getValueInfo(ValueGUID);
586 if (!CalleeInfo) {
597587 return nullptr; // This function does not have a summary
598588 }
599589 auto Summary =
600 llvm::find_if(CalleeInfoList->second,
590 llvm::find_if(CalleeInfo.getSummaryList(),
601591 [&](const std::unique_ptr &Summary) {
602592 return Summary->modulePath() == ModuleId;
603593 });
604 if (Summary == CalleeInfoList->second.end())
594 if (Summary == CalleeInfo.getSummaryList().end())
605595 return nullptr;
606596 return Summary->get();
607597 }
200200 for (auto &FSum : FSums) {
201201 GlobalValueSummary::GVFlags GVFlags(GlobalValue::ExternalLinkage, false,
202202 false);
203 Elem.push_back(llvm::make_unique(
203 Elem.SummaryList.push_back(llvm::make_unique(
204204 GVFlags, 0, ArrayRef{},
205205 ArrayRef{}, std::move(FSum.TypeTests),
206206 std::move(FSum.TypeTestAssumeVCalls),
212212 static void output(IO &io, GlobalValueSummaryMapTy &V) {
213213 for (auto &P : V) {
214214 std::vector FSums;
215 for (auto &Sum : P.second) {
215 for (auto &Sum : P.second.SummaryList) {
216216 if (auto *FSum = dyn_cast(Sum.get()))
217217 FSums.push_back(FunctionSummaryYaml{
218218 FSum->type_tests(), FSum->type_test_assume_vcalls(),
3636 // Walk through the operands of a given User via worklist iteration and populate
3737 // the set of GlobalValue references encountered. Invoked either on an
3838 // Instruction or a GlobalVariable (which walks its initializer).
39 static void findRefEdges(const User *CurUser, SetVector &RefEdges,
39 static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
40 SetVector &RefEdges,
4041 SmallPtrSet &Visited) {
4142 SmallVector Worklist;
4243 Worklist.push_back(CurUser);
6061 // the reference set unless it is a callee. Callees are handled
6162 // specially by WriteFunction and are added to a separate list.
6263 if (!(CS && CS.isCallee(&OI)))
63 RefEdges.insert(GV);
64 RefEdges.insert(Index.getOrInsertValueInfo(GV));
6465 continue;
6566 }
6667 Worklist.push_back(Operand);
197198 if (isa(I))
198199 continue;
199200 ++NumInsts;
200 findRefEdges(&I, RefEdges, Visited);
201 findRefEdges(Index, &I, RefEdges, Visited);
201202 auto CS = ImmutableCallSite(&I);
202203 if (!CS)
203204 continue;
238239 // to record the call edge to the alias in that case. Eventually
239240 // an alias summary will be created to associate the alias and
240241 // aliasee.
241 CallGraphEdges[cast(CalledValue)].updateHotness(Hotness);
242 CallGraphEdges[Index.getOrInsertValueInfo(
243 cast(CalledValue))]
244 .updateHotness(Hotness);
242245 } else {
243246 // Skip inline assembly calls.
244247 if (CI && CI->isInlineAsm())
253256 ICallAnalysis.getPromotionCandidatesForInstruction(
254257 &I, NumVals, TotalCount, NumCandidates);
255258 for (auto &Candidate : CandidateProfileData)
256 CallGraphEdges[Candidate.Value].updateHotness(
257 getHotness(Candidate.Count, PSI));
259 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
260 .updateHotness(getHotness(Candidate.Count, PSI));
258261 }
259262 }
260263
261264 // Explicit add hot edges to enforce importing for designated GUIDs for
262265 // sample PGO, to enable the same inlines as the profiled optimized binary.
263266 for (auto &I : F.getImportGUIDs())
264 CallGraphEdges[I].updateHotness(CalleeInfo::HotnessType::Hot);
267 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
268 CalleeInfo::HotnessType::Hot);
265269
266270 bool NonRenamableLocal = isNonRenamableLocal(F);
267271 bool NotEligibleForImport =
287291 DenseSet &CantBePromoted) {
288292 SetVector RefEdges;
289293 SmallPtrSet Visited;
290 findRefEdges(&V, RefEdges, Visited);
294 findRefEdges(Index, &V, RefEdges, Visited);
291295 bool NonRenamableLocal = isNonRenamableLocal(V);
292296 GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
293297 /* LiveRoot = */ false);
316320
317321 // Set LiveRoot flag on entries matching the given value name.
318322 static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
319 auto SummaryList =
320 Index.findGlobalValueSummaryList(GlobalValue::getGUID(Name));
321 if (SummaryList == Index.end())
322 return;
323 for (auto &Summary : SummaryList->second)
324 Summary->setLiveRoot();
323 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
324 for (auto &Summary : VI.getSummaryList())
325 Summary->setLiveRoot();
325326 }
326327
327328 ModuleSummaryIndex llvm::buildModuleSummaryIndex(
445446 }
446447
447448 for (auto &GlobalList : Index) {
448 assert(GlobalList.second.size() == 1 &&
449 // Ignore entries for references that are undefined in the current module.
450 if (GlobalList.second.SummaryList.empty())
451 continue;
452
453 assert(GlobalList.second.SummaryList.size() == 1 &&
449454 "Expected module's index to have one summary per GUID");
450 auto &Summary = GlobalList.second[0];
455 auto &Summary = GlobalList.second.SummaryList[0];
451456 bool AllRefsCanBeExternallyReferenced =
452457 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
453 return !CantBePromoted.count(VI.getValue()->getGUID());
458 return !CantBePromoted.count(VI.getGUID());
454459 });
455460 if (!AllRefsCanBeExternallyReferenced) {
456461 Summary->setNotEligibleToImport();
460465 if (auto *FuncSummary = dyn_cast(Summary.get())) {
461466 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
462467 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
463 auto GUID = Edge.first.isGUID() ? Edge.first.getGUID()
464 : Edge.first.getValue()->getGUID();
465 return !CantBePromoted.count(GUID);
468 return !CantBePromoted.count(Edge.first.getGUID());
466469 });
467470 if (!AllCallsCanBeExternallyReferenced)
468471 Summary->setNotEligibleToImport();
693693 /// Used to enable on-demand parsing of the VST.
694694 uint64_t VSTOffset = 0;
695695
696 // Map to save ValueId to GUID association that was recorded in the
696 // Map to save ValueId to ValueInfo association that was recorded in the
697697 // ValueSymbolTable. It is used after the VST is parsed to convert
698698 // call graph edges read from the function summary from referencing
699 // callees by their ValueId to using the GUID instead, which is how
699 // callees by their ValueId to using the ValueInfo instead, which is how
700700 // they are recorded in the summary index being built.
701 // We save a second GUID which is the same as the first one, but ignoring the
702 // linkage, i.e. for value other than local linkage they are identical.
703 DenseMap>
704 ValueIdToCallGraphGUIDMap;
701 // We save a GUID which refers to the same global as the ValueInfo, but
702 // ignoring the linkage, i.e. for values other than local linkage they are
703 // identical.
704 DenseMap>
705 ValueIdToValueInfoMap;
705706
706707 /// Map populated during module path string table parsing, from the
707708 /// module ID to a string reference owned by the index's module
741742 Error parseEntireSummary();
742743 Error parseModuleStringTable();
743744
744 std::pair
745 getGUIDFromValueId(unsigned ValueId);
745 std::pair
746 getValueInfoFromValueId(unsigned ValueId);
746747
747748 ModulePathStringTableTy::iterator addThisModulePath();
748749 };
46964697 return TheIndex.addModulePath(ModulePath, ModuleId);
46974698 }
46984699
4699 std::pair
4700 ModuleSummaryIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) {
4701 auto VGI = ValueIdToCallGraphGUIDMap.find(ValueId);
4702 assert(VGI != ValueIdToCallGraphGUIDMap.end());
4703 return VGI->second;
4700 std::pair
4701 ModuleSummaryIndexBitcodeReader::getValueInfoFromValueId(unsigned ValueId) {
4702 auto VGI = ValueIdToValueInfoMap[ValueId];
4703 assert(VGI.first);
4704 return VGI;
47044705 }
47054706
47064707 void ModuleSummaryIndexBitcodeReader::setValueGUID(
47154716 if (PrintSummaryGUIDs)
47164717 dbgs() << "GUID " << ValueGUID << "(" << OriginalNameID << ") is "
47174718 << ValueName << "\n";
4718 ValueIdToCallGraphGUIDMap[ValueID] =
4719 std::make_pair(ValueGUID, OriginalNameID);
4719 ValueIdToValueInfoMap[ValueID] =
4720 std::make_pair(TheIndex.getOrInsertValueInfo(ValueGUID), OriginalNameID);
47204721 }
47214722
47224723 // Specialized value symbol table parser used when reading module index
47944795 GlobalValue::GUID RefGUID = Record[1];
47954796 // The "original name", which is the second value of the pair will be
47964797 // overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index.
4797 ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID);
4798 ValueIdToValueInfoMap[ValueID] =
4799 std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
47984800 break;
47994801 }
48004802 }
49394941 std::vector Ret;
49404942 Ret.reserve(Record.size());
49414943 for (uint64_t RefValueId : Record)
4942 Ret.push_back(getGUIDFromValueId(RefValueId).first);
4944 Ret.push_back(getValueInfoFromValueId(RefValueId).first);
49434945 return Ret;
49444946 }
49454947
49494951 Ret.reserve(Record.size());
49504952 for (unsigned I = 0, E = Record.size(); I != E; ++I) {
49514953 CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown;
4952 GlobalValue::GUID CalleeGUID = getGUIDFromValueId(Record[I]).first;
4954 ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
49534955 if (IsOldProfileFormat) {
49544956 I += 1; // Skip old callsitecount field
49554957 if (HasProfile)
49564958 I += 1; // Skip old profilecount field
49574959 } else if (HasProfile)
49584960 Hotness = static_cast(Record[++I]);
4959 Ret.push_back(FunctionSummary::EdgeTy{CalleeGUID, CalleeInfo{Hotness}});
4961 Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}});
49604962 }
49614963 return Ret;
49624964 }
50265028 case bitc::FS_VALUE_GUID: { // [valueid, refguid]
50275029 uint64_t ValueID = Record[0];
50285030 GlobalValue::GUID RefGUID = Record[1];
5029 ValueIdToCallGraphGUIDMap[ValueID] = std::make_pair(RefGUID, RefGUID);
5031 ValueIdToValueInfoMap[ValueID] =
5032 std::make_pair(TheIndex.getOrInsertValueInfo(RefGUID), RefGUID);
50305033 break;
50315034 }
50325035 // FS_PERMODULE: [valueid, flags, instcount, numrefs, numrefs x valueid,
50675070 PendingTypeCheckedLoadVCalls.clear();
50685071 PendingTypeTestAssumeConstVCalls.clear();
50695072 PendingTypeCheckedLoadConstVCalls.clear();
5070 auto GUID = getGUIDFromValueId(ValueID);
5073 auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID);
50715074 FS->setModulePath(addThisModulePath()->first());
5072 FS->setOriginalName(GUID.second);
5073 TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
5075 FS->setOriginalName(VIAndOriginalGUID.second);
5076 TheIndex.addGlobalValueSummary(VIAndOriginalGUID.first, std::move(FS));
50745077 break;
50755078 }
50765079 // FS_ALIAS: [valueid, flags, valueid]
50905093 // ownership.
50915094 AS->setModulePath(addThisModulePath()->first());
50925095
5093 GlobalValue::GUID AliaseeGUID = getGUIDFromValueId(AliaseeID).first;
5096 GlobalValue::GUID AliaseeGUID =
5097 getValueInfoFromValueId(AliaseeID).first.getGUID();
50945098 auto AliaseeInModule =
50955099 TheIndex.findSummaryInModule(AliaseeGUID, ModulePath);
50965100 if (!AliaseeInModule)
50975101 return error("Alias expects aliasee summary to be parsed");
50985102 AS->setAliasee(AliaseeInModule);
50995103
5100 auto GUID = getGUIDFromValueId(ValueID);
5104 auto GUID = getValueInfoFromValueId(ValueID);
51015105 AS->setOriginalName(GUID.second);
51025106 TheIndex.addGlobalValueSummary(GUID.first, std::move(AS));
51035107 break;
51115115 makeRefList(ArrayRef(Record).slice(2));
51125116 auto FS = llvm::make_unique(Flags, std::move(Refs));
51135117 FS->setModulePath(addThisModulePath()->first());
5114 auto GUID = getGUIDFromValueId(ValueID);
5118 auto GUID = getValueInfoFromValueId(ValueID);
51155119 FS->setOriginalName(GUID.second);
51165120 TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
51175121 break;
51385142 std::vector Edges = makeCallList(
51395143 ArrayRef(Record).slice(CallGraphEdgeStartIndex),
51405144 IsOldProfileFormat, HasProfile);
5141 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5145 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
51425146 auto FS = llvm::make_unique(
51435147 Flags, InstCount, std::move(Refs), std::move(Edges),
51445148 std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls),
51515155 PendingTypeTestAssumeConstVCalls.clear();
51525156 PendingTypeCheckedLoadConstVCalls.clear();
51535157 LastSeenSummary = FS.get();
5154 LastSeenGUID = GUID;
5158 LastSeenGUID = VI.getGUID();
51555159 FS->setModulePath(ModuleIdMap[ModuleId]);
5156 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
5160 TheIndex.addGlobalValueSummary(VI, std::move(FS));
51575161 break;
51585162 }
51595163 // FS_COMBINED_ALIAS: [valueid, modid, flags, valueid]
51695173 LastSeenSummary = AS.get();
51705174 AS->setModulePath(ModuleIdMap[ModuleId]);
51715175
5172 auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first;
5176 auto AliaseeGUID =
5177 getValueInfoFromValueId(AliaseeValueId).first.getGUID();
51735178 auto AliaseeInModule =
51745179 TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath());
51755180 if (!AliaseeInModule)
51765181 return error("Alias expects aliasee summary to be parsed");
51775182 AS->setAliasee(AliaseeInModule);
51785183
5179 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5180 LastSeenGUID = GUID;
5181 TheIndex.addGlobalValueSummary(GUID, std::move(AS));
5184 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
5185 LastSeenGUID = VI.getGUID();
5186 TheIndex.addGlobalValueSummary(VI, std::move(AS));
51825187 break;
51835188 }
51845189 // FS_COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid]
51925197 auto FS = llvm::make_unique(Flags, std::move(Refs));
51935198 LastSeenSummary = FS.get();
51945199 FS->setModulePath(ModuleIdMap[ModuleId]);
5195 GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
5196 LastSeenGUID = GUID;
5197 TheIndex.addGlobalValueSummary(GUID, std::move(FS));
5200 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
5201 LastSeenGUID = VI.getGUID();
5202 TheIndex.addGlobalValueSummary(VI, std::move(FS));
51985203 break;
51995204 }
52005205 // FS_COMBINED_ORIGINAL_NAME: [original_name]
155155 return;
156156 for (const auto &GUIDSummaryLists : *Index)
157157 // Examine all summaries for this GUID.
158 for (auto &Summary : GUIDSummaryLists.second)
158 for (auto &Summary : GUIDSummaryLists.second.SummaryList)
159159 if (auto FS = dyn_cast(Summary.get()))
160160 // For each call in the function summary, see if the call
161161 // is to a GUID (which means it is for an indirect call,
162162 // otherwise we would have a Value for it). If so, synthesize
163163 // a value id.
164164 for (auto &CallEdge : FS->calls())
165 if (CallEdge.first.isGUID())
165 if (!CallEdge.first.getValue())
166166 assignValueId(CallEdge.first.getGUID());
167167 }
168168
303303 }
304304 // Helper to get the valueId for the type of value recorded in VI.
305305 unsigned getValueId(ValueInfo VI) {
306 if (VI.isGUID())
306 if (!VI.getValue())
307307 return getValueId(VI.getGUID());
308308 return VE.getValueID(VI.getValue());
309309 }
357357 Callback(Summary);
358358 } else {
359359 for (auto &Summaries : Index)
360 for (auto &Summary : Summaries.second)
360 for (auto &Summary : Summaries.second.SummaryList)
361361 Callback({Summaries.first, Summary.get()});
362362 }
363363 }
32693269 void ModuleBitcodeWriter::writeModuleLevelReferences(
32703270 const GlobalVariable &V, SmallVector &NameVals,
32713271 unsigned FSModRefsAbbrev) {
3272 auto Summaries =
3273 Index->findGlobalValueSummaryList(GlobalValue::getGUID(V.getName()));
3274 if (Summaries == Index->end()) {
3272 auto VI = Index->getValueInfo(GlobalValue::getGUID(V.getName()));
3273 if (!VI || VI.getSummaryList().empty()) {
32753274 // Only declarations should not have a summary (a declaration might however
32763275 // have a summary if the def was in module level asm).
32773276 assert(V.isDeclaration());
32783277 return;
32793278 }
3280 auto *Summary = Summaries->second.front().get();
3279 auto *Summary = VI.getSummaryList()[0].get();
32813280 NameVals.push_back(VE.getValueID(&V));
32823281 GlobalVarSummary *VS = cast(Summary);
32833282 NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
33663365 if (!F.hasName())
33673366 report_fatal_error("Unexpected anonymous function when writing summary");
33683367
3369 auto Summaries =
3370 Index->findGlobalValueSummaryList(GlobalValue::getGUID(F.getName()));
3371 if (Summaries == Index->end()) {
3368 ValueInfo VI = Index->getValueInfo(GlobalValue::getGUID(F.getName()));
3369 if (!VI || VI.getSummaryList().empty()) {
33723370 // Only declarations should not have a summary (a declaration might
33733371 // however have a summary if the def was in module level asm).
33743372 assert(F.isDeclaration());
33753373 continue;
33763374 }
3377 auto *Summary = Summaries->second.front().get();
3375 auto *Summary = VI.getSummaryList()[0].get();
33783376 writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F),
33793377 FSCallsAbbrev, FSCallsProfileAbbrev, F);
33803378 }
2121 StringRef ModulePath, GVSummaryMapTy &GVSummaryMap) const {
2222 for (auto &GlobalList : *this) {
2323 auto GUID = GlobalList.first;
24 for (auto &GlobSummary : GlobalList.second) {
24 for (auto &GlobSummary : GlobalList.second.SummaryList) {
2525 auto *Summary = dyn_cast_or_null(GlobSummary.get());
2626 if (!Summary)
2727 // Ignore global variable, focus on functions
3939 StringMap &ModuleToDefinedGVSummaries) const {
4040 for (auto &GlobalList : *this) {
4141 auto GUID = GlobalList.first;
42 for (auto &Summary : GlobalList.second) {
42 for (auto &Summary : GlobalList.second.SummaryList) {
4343 ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get();
4444 }
4545 }
4848 GlobalValueSummary *
4949 ModuleSummaryIndex::getGlobalValueSummary(uint64_t ValueGUID,
5050 bool PerModuleIndex) const {
51 auto SummaryList = findGlobalValueSummaryList(ValueGUID);
52 assert(SummaryList != end() && "GlobalValue not found in index");
53 assert((!PerModuleIndex || SummaryList->second.size() == 1) &&
51 auto VI = getValueInfo(ValueGUID);
52 assert(VI && "GlobalValue not found in index");
53 assert((!PerModuleIndex || VI.getSummaryList().size() == 1) &&
5454 "Expected a single entry per global value in per-module index");
55 auto &Summary = SummaryList->second[0];
55 auto &Summary = VI.getSummaryList()[0];
5656 return Summary.get();
5757 }
273273 // when needed.
274274 DenseSet GlobalInvolvedWithAlias;
275275 for (auto &I : Index)
276 for (auto &S : I.second)
276 for (auto &S : I.second.SummaryList)
277277 if (auto AS = dyn_cast(S.get()))
278278 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
279279
280280 for (auto &I : Index)
281 thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias,
282 isPrevailing, recordNewLinkage);
281 thinLTOResolveWeakForLinkerGUID(I.second.SummaryList, I.first,
282 GlobalInvolvedWithAlias, isPrevailing,
283 recordNewLinkage);
283284 }
284285
285286 static void thinLTOInternalizeAndPromoteGUID(
300301 ModuleSummaryIndex &Index,
301302 function_ref isExported) {
302303 for (auto &I : Index)
303 thinLTOInternalizeAndPromoteGUID(I.second, I.first, isExported);
304 thinLTOInternalizeAndPromoteGUID(I.second.SummaryList, I.first, isExported);
304305 }
305306
306307 // Requires a destructor for std::vector.
118118 };
119119
120120 for (auto &I : Index) {
121 if (HasMultipleCopies(I.second))
122 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second);
121 if (HasMultipleCopies(I.second.SummaryList))
122 PrevailingCopy[I.first] =
123 getFirstDefinitionForLinker(I.second.SummaryList);
123124 }
124125 }
125126
116116 /// - [insert you fancy metric here]
117117 static const GlobalValueSummary *
118118 selectCallee(const ModuleSummaryIndex &Index,
119 const GlobalValueSummaryList &CalleeSummaryList,
119 ArrayRef> CalleeSummaryList,
120120 unsigned Threshold, StringRef CallerModulePath) {
121121 auto It = llvm::find_if(
122122 CalleeSummaryList,
167167 return cast(It->get());
168168 }
169169
170 /// Return the summary for the function \p GUID that fits the \p Threshold, or
171 /// null if there's no match.
172 static const GlobalValueSummary *selectCallee(GlobalValue::GUID GUID,
173 unsigned Threshold,
174 const ModuleSummaryIndex &Index,
175 StringRef CallerModulePath) {
176 auto CalleeSummaryList = Index.findGlobalValueSummaryList(GUID);
177 if (CalleeSummaryList == Index.end())
178 return nullptr; // This function does not have a summary
179 return selectCallee(Index, CalleeSummaryList->second, Threshold,
180 CallerModulePath);
181 }
182
183170 using EdgeInfo = std::tuple
184171 GlobalValue::GUID>;
185172
193180 FunctionImporter::ImportMapTy &ImportList,
194181 StringMap *ExportLists = nullptr) {
195182 for (auto &Edge : Summary.calls()) {
196 auto GUID = Edge.first.getGUID();
197 DEBUG(dbgs() << " edge -> " << GUID << " Threshold:" << Threshold << "\n");
198
199 if (Index.findGlobalValueSummaryList(GUID) == Index.end()) {
183 ValueInfo VI = Edge.first;
184 DEBUG(dbgs() << " edge -> " << VI.getGUID() << " Threshold:" << Threshold
185 << "\n");
186
187 if (VI.getSummaryList().empty()) {
200188 // For SamplePGO, the indirect call targets for local functions will
201189 // have its original name annotated in profile. We try to find the
202190 // corresponding PGOFuncName as the GUID.
203 GUID = Index.getGUIDFromOriginalID(GUID);
191 auto GUID = Index.getGUIDFromOriginalID(VI.getGUID());
204192 if (GUID == 0)
205193 continue;
206 }
207
208 if (DefinedGVSummaries.count(GUID)) {
194 VI = Index.getValueInfo(GUID);
195 if (!VI)
196 continue;
197 }
198
199 if (DefinedGVSummaries.count(VI.getGUID())) {
209200 DEBUG(dbgs() << "ignored! Target already in destination module.\n");
210201 continue;
211202 }
221212 const auto NewThreshold =
222213 Threshold * GetBonusMultiplier(Edge.second.Hotness);
223214
224 auto *CalleeSummary =
225 selectCallee(GUID, NewThreshold, Index, Summary.modulePath());
215 auto *CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold,
216 Summary.modulePath());
226217 if (!CalleeSummary) {
227218 DEBUG(dbgs() << "ignored! No qualifying callee with summary found.\n");
228219 continue;
254245 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
255246
256247 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
257 auto &ProcessedThreshold = ImportList[ExportModulePath][GUID];
248 auto &ProcessedThreshold = ImportList[ExportModulePath][VI.getGUID()];
258249 /// Since the traversal of the call graph is DFS, we can revisit a function
259250 /// a second time with a higher threshold. In this case, it is added back to
260251 /// the worklist with the new threshold.
270261 // Make exports in the source module.
271262 if (ExportLists) {
272263 auto &ExportList = (*ExportLists)[ExportModulePath];
273 ExportList.insert(GUID);
264 ExportList.insert(VI.getGUID());
274265 if (!PreviouslyImported) {
275266 // This is the first time this function was exported from its source
276267 // module, so mark all functions and globals it references as exported
290281 }
291282
292283 // Insert the newly imported function to the worklist.
293 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, GUID);
284 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID());
294285 }
295286 }
296287
430421 if (GUIDPreservedSymbols.empty())
431422 // Don't do anything when nothing is live, this is friendly with tests.
432423 return DenseSet();
433 DenseSet LiveSymbols = GUIDPreservedSymbols;
434 SmallVector Worklist;
435 Worklist.reserve(LiveSymbols.size() * 2);
436 for (auto GUID : LiveSymbols) {
437 DEBUG(dbgs() << "Live root: " << GUID << "\n");
438 Worklist.push_back(GUID);
424 DenseSet LiveSymbols;
425 SmallVector Worklist;
426 Worklist.reserve(GUIDPreservedSymbols.size() * 2);
427 for (auto GUID : GUIDPreservedSymbols) {
428 ValueInfo VI = Index.getValueInfo(GUID);
429 if (!VI)
430 continue;
431 DEBUG(dbgs() << "Live root: " << VI.getGUID() << "\n");
432 LiveSymbols.insert(VI);
433 Worklist.push_back(VI);
439434 }
440435 // Add values flagged in the index as live roots to the worklist.
441436 for (const auto &Entry : Index) {
442437 bool IsLiveRoot = llvm::any_of(
443 Entry.second,
438 Entry.second.SummaryList,
444439 [&](const std::unique_ptr &Summary) {
445440 return Summary->liveRoot();
446441 });
447442 if (!IsLiveRoot)
448443 continue;
449444 DEBUG(dbgs() << "Live root (summary): " << Entry.first << "\n");
450 Worklist.push_back(Entry.first);
445 Worklist.push_back(ValueInfo(&Entry));
451446 }
452447
453448 while (!Worklist.empty()) {
454 auto GUID = Worklist.pop_back_val();
455 auto It = Index.findGlobalValueSummaryList(GUID);
456 if (It == Index.end()) {
457 DEBUG(dbgs() << "Not in index: " << GUID << "\n");
458 continue;
459 }
449 auto VI = Worklist.pop_back_val();
460450
461451 // FIXME: we should only make the prevailing copy live here
462 for (auto &Summary : It->second) {
452 for (auto &Summary : VI.getSummaryList()) {
463453 for (auto Ref : Summary->refs()) {
464 auto RefGUID = Ref.getGUID();
465 if (LiveSymbols.insert(RefGUID).second) {
466 DEBUG(dbgs() << "Marking live (ref): " << RefGUID << "\n");
467 Worklist.push_back(RefGUID);
454 if (LiveSymbols.insert(Ref).second) {
455 DEBUG(dbgs() << "Marking live (ref): " << Ref.getGUID() << "\n");
456 Worklist.push_back(Ref);
468457 }
469458 }
470459 if (auto *FS = dyn_cast(Summary.get())) {
471460 for (auto Call : FS->calls()) {
472 auto CallGUID = Call.first.getGUID();
473 if (LiveSymbols.insert(CallGUID).second) {
474 DEBUG(dbgs() << "Marking live (call): " << CallGUID << "\n");
475 Worklist.push_back(CallGUID);
461 if (LiveSymbols.insert(Call.first).second) {
462 DEBUG(dbgs() << "Marking live (call): " << Call.first.getGUID()
463 << "\n");
464 Worklist.push_back(Call.first);
476465 }
477466 }
478467 }
479468 if (auto *AS = dyn_cast(Summary.get())) {
480469 auto AliaseeGUID = AS->getAliasee().getOriginalName();
481 if (LiveSymbols.insert(AliaseeGUID).second) {
470 ValueInfo AliaseeVI = Index.getValueInfo(AliaseeGUID);
471 if (AliaseeVI && LiveSymbols.insert(AliaseeVI).second) {
482472 DEBUG(dbgs() << "Marking live (alias): " << AliaseeGUID << "\n");
483 Worklist.push_back(AliaseeGUID);
473 Worklist.push_back(AliaseeVI);
484474 }
485475 }
486476 }
489479 DeadSymbols.reserve(
490480 std::min(Index.size(), Index.size() - LiveSymbols.size()));
491481 for (auto &Entry : Index) {
492 auto GUID = Entry.first;
493 if (!LiveSymbols.count(GUID)) {
494 DEBUG(dbgs() << "Marking dead: " << GUID << "\n");
495 DeadSymbols.insert(GUID);
482 if (!LiveSymbols.count(ValueInfo(&Entry))) {
483 DEBUG(dbgs() << "Marking dead: " << Entry.first << "\n");
484 DeadSymbols.insert(Entry.first);
496485 }
497486 }
498487 DEBUG(dbgs() << LiveSymbols.size() << " symbols Live, and "
824813 // is only enabled when testing importing via the 'opt' tool, which does
825814 // not do the ThinLink that would normally determine what values to promote.
826815 for (auto &I : *Index) {
827 for (auto &S : I.second) {
816 for (auto &S : I.second.SummaryList) {
828817 if (GlobalValue::isLocalLinkage(S->linkage()))
829818 S->setLinkage(GlobalValue::ExternalLinkage);
830819 }
14391439 }
14401440
14411441 for (auto &P : *ExportSummary) {
1442 for (auto &S : P.second) {
1442 for (auto &S : P.second.SummaryList) {
14431443 auto *FS = dyn_cast(S.get());
14441444 if (!FS)
14451445 continue;
13211321 }
13221322
13231323 for (auto &P : *ExportSummary) {
1324 for (auto &S : P.second) {
1324 for (auto &S : P.second.SummaryList) {
13251325 auto *FS = dyn_cast(S.get());
13261326 if (!FS)
13271327 continue;
299299 // does not do the ThinLink that would normally determine what values to
300300 // promote.
301301 for (auto &I : *Index) {
302 for (auto &S : I.second) {
302 for (auto &S : I.second.SummaryList) {
303303 if (GlobalValue::isLocalLinkage(S->linkage()))
304304 S->setLinkage(GlobalValue::ExternalLinkage);
305305 }
283283
284284 unsigned Calls = 0, Refs = 0, Functions = 0, Alias = 0, Globals = 0;
285285 for (auto &Summaries : *Index) {
286 for (auto &Summary : Summaries.second) {
286 for (auto &Summary : Summaries.second.SummaryList) {
287287 Refs += Summary->refs().size();
288288 if (auto *FuncSummary = dyn_cast(Summary.get())) {
289289 Functions++;