llvm.org GIT mirror llvm / bdb305a
Re-land "[ThinLTO] Add call edges' relative block frequency to per-module summary." It was reverted after buildbot regressions. Original commit message: This allows relative block frequency of call edges to be passed to the thinlink stage where it will be used to compute synthetic entry counts of functions. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@323460 91177308-0d34-0410-b5e6-96231b3b80d8 Easwaran Raman 2 years ago
7 changed file(s) with 94 addition(s) and 32 deletion(s). Raw diff Collapse all Expand all
255255 // strings in strtab.
256256 // [n * name]
257257 FS_CFI_FUNCTION_DECLS = 18,
258 // Per-module summary that also adds relative block frequency to callee info.
259 // PERMODULE_RELBF: [valueid, flags, instcount, numrefs,
260 // numrefs x valueid,
261 // n x (valueid, relblockfreq)]
262 FS_PERMODULE_RELBF = 19,
258263 };
259264
260265 enum MetadataCodes {
2424 #include "llvm/ADT/StringRef.h"
2525 #include "llvm/IR/GlobalValue.h"
2626 #include "llvm/IR/Module.h"
27 #include "llvm/Support/MathExtras.h"
2728 #include
2829 #include
2930 #include
5354 Hot = 3,
5455 Critical = 4
5556 };
56 HotnessType Hotness = HotnessType::Unknown;
57
58 CalleeInfo() = default;
59 explicit CalleeInfo(HotnessType Hotness) : Hotness(Hotness) {}
57
58 // The size of the bit-field might need to be adjusted if more values are
59 // added to HotnessType enum.
60 uint32_t Hotness : 3;
61 uint32_t RelBlockFreq : 29;
62 static constexpr uint64_t MaxRelBlockFreq = (1 << 29) - 1;
63
64 CalleeInfo()
65 : Hotness(static_cast(HotnessType::Unknown)), RelBlockFreq(0) {}
66 explicit CalleeInfo(HotnessType Hotness, uint64_t RelBF)
67 : Hotness(static_cast(Hotness)), RelBlockFreq(RelBF) {}
6068
6169 void updateHotness(const HotnessType OtherHotness) {
62 Hotness = std::max(Hotness, OtherHotness);
70 Hotness = std::max(Hotness, static_cast(OtherHotness));
71 }
72
73 HotnessType getHotness() const { return HotnessType(Hotness); }
74
75 // When there are multiple edges between the same (caller, callee) pair, the
76 // relative block frequencies are summed up.
77 void updateRelBlockFreq(uint64_t RBF) {
78 uint64_t Sum = SaturatingAdd(RelBlockFreq, RBF);
79 Sum = std::min(Sum, uint64_t(MaxRelBlockFreq));
80 RelBlockFreq = static_cast(Sum);
6381 }
6482 };
6583
272272 // to record the call edge to the alias in that case. Eventually
273273 // an alias summary will be created to associate the alias and
274274 // aliasee.
275 CallGraphEdges[Index.getOrInsertValueInfo(
276 cast(CalledValue))]
277 .updateHotness(Hotness);
275 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
276 cast(CalledValue))];
277 ValueInfo.updateHotness(Hotness);
278 // Add the relative block frequency to CalleeInfo if there is no profile
279 // information.
280 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
281 auto BBFreq = BFI->getBlockFreq(&BB).getFrequency();
282 // FIXME: This might need some scaling to prevent BBFreq values from
283 // being rounded down to 0.
284 auto EntryFreq = BFI->getEntryFreq();
285 // Block frequencies can be directly set for a block and so we need to
286 // handle the case of entry frequency being 0.
287 if (EntryFreq)
288 BBFreq /= EntryFreq;
289 else
290 BBFreq = 0;
291 ValueInfo.updateRelBlockFreq(BBFreq);
292 }
278293 } else {
279294 // Skip inline assembly calls.
280295 if (CI && CI->isInlineAsm())
742742 std::vector makeRefList(ArrayRef Record);
743743 std::vector makeCallList(ArrayRef Record,
744744 bool IsOldProfileFormat,
745 bool HasProfile);
745 bool HasProfile,
746 bool HasRelBF);
746747 Error parseEntireSummary(unsigned ID);
747748 Error parseModuleStringTable();
748749
50465047 return Ret;
50475048 }
50485049
5049 std::vector ModuleSummaryIndexBitcodeReader::makeCallList(
5050 ArrayRef Record, bool IsOldProfileFormat, bool HasProfile) {
5050 std::vector
5051 ModuleSummaryIndexBitcodeReader::makeCallList(ArrayRef Record,
5052 bool IsOldProfileFormat,
5053 bool HasProfile, bool HasRelBF) {
50515054 std::vector Ret;
50525055 Ret.reserve(Record.size());
50535056 for (unsigned I = 0, E = Record.size(); I != E; ++I) {
50545057 CalleeInfo::HotnessType Hotness = CalleeInfo::HotnessType::Unknown;
5058 uint64_t RelBF = 0;
50555059 ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
50565060 if (IsOldProfileFormat) {
50575061 I += 1; // Skip old callsitecount field
50595063 I += 1; // Skip old profilecount field
50605064 } else if (HasProfile)
50615065 Hotness = static_cast(Record[++I]);
5062 Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo{Hotness}});
5066 else if (HasRelBF)
5067 RelBF = Record[++I];
5068 Ret.push_back(FunctionSummary::EdgeTy{Callee, CalleeInfo(Hotness, RelBF)});
50635069 }
50645070 return Ret;
50655071 }
51385144 // FS_PERMODULE_PROFILE: [valueid, flags, instcount, fflags, numrefs,
51395145 // numrefs x valueid,
51405146 // n x (valueid, hotness)]
5147 // FS_PERMODULE_RELBF: [valueid, flags, instcount, fflags, numrefs,
5148 // numrefs x valueid,
5149 // n x (valueid, relblockfreq)]
51415150 case bitc::FS_PERMODULE:
5151 case bitc::FS_PERMODULE_RELBF:
51425152 case bitc::FS_PERMODULE_PROFILE: {
51435153 unsigned ValueID = Record[0];
51445154 uint64_t RawFlags = Record[1];
51645174 std::vector Refs = makeRefList(
51655175 ArrayRef(Record).slice(RefListStartIndex, NumRefs));
51665176 bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE);
5177 bool HasRelBF = (BitCode == bitc::FS_PERMODULE_RELBF);
51675178 std::vector Calls = makeCallList(
51685179 ArrayRef(Record).slice(CallGraphEdgeStartIndex),
5169 IsOldProfileFormat, HasProfile);
5180 IsOldProfileFormat, HasProfile, HasRelBF);
51705181 auto FS = llvm::make_unique(
51715182 Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
51725183 std::move(Calls), std::move(PendingTypeTests),
52585269 bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE);
52595270 std::vector Edges = makeCallList(
52605271 ArrayRef(Record).slice(CallGraphEdgeStartIndex),
5261 IsOldProfileFormat, HasProfile);
5272 IsOldProfileFormat, HasProfile, false);
52625273 ValueInfo VI = getValueInfoFromValueId(ValueID).first;
52635274 auto FS = llvm::make_unique(
52645275 Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
8585 cl::desc("Number of metadatas above which we emit an index "
8686 "to enable lazy-loading"));
8787
88 cl::opt WriteRelBFToSummary(
89 "write-relbf-to-summary", cl::Hidden, cl::init(false),
90 cl::desc("Write relative block frequency to function summary "));
8891 namespace {
8992
9093 /// These are manifest constants used by the bitcode writer. They do not need to
33773380 NameVals.push_back(getValueId(ECI.first));
33783381 if (HasProfileData)
33793382 NameVals.push_back(static_cast(ECI.second.Hotness));
3383 else if (WriteRelBFToSummary)
3384 NameVals.push_back(ECI.second.RelBlockFreq);
33803385 }
33813386
33823387 unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
33833388 unsigned Code =
3384 (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE);
3389 (HasProfileData ? bitc::FS_PERMODULE_PROFILE
3390 : (WriteRelBFToSummary ? bitc::FS_PERMODULE_RELBF
3391 : bitc::FS_PERMODULE));
33853392
33863393 // Emit the finished record.
33873394 Stream.EmitRecord(Code, NameVals, FSAbbrev);
34473454 ArrayRef{GVI.second, GVI.first});
34483455 }
34493456
3450 // Abbrev for FS_PERMODULE.
3457 // Abbrev for FS_PERMODULE_PROFILE.
34513458 auto Abbv = std::make_shared();
3452 Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
3453 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
3454 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
3455 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
3456 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
3457 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
3458 // numrefs x valueid, n x (valueid)
3459 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3460 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3461 unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
3462
3463 // Abbrev for FS_PERMODULE_PROFILE.
3464 Abbv = std::make_shared();
34653459 Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
34663460 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
34673461 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
34723466 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
34733467 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
34743468 unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(std::move(Abbv));
3469
3470 // Abbrev for FS_PERMODULE or FS_PERMODULE_RELBF.
3471 Abbv = std::make_shared();
3472 if (WriteRelBFToSummary)
3473 Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
3474 else
3475 Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
3476 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
3477 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
3478 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
3479 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
3480 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
3481 // numrefs x valueid, n x (valueid [, rel_block_freq])
3482 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3483 Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3484 unsigned FSCallsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
34753485
34763486 // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
34773487 Abbv = std::make_shared();
36743684
36753685 bool HasProfileData = false;
36763686 for (auto &EI : FS->calls()) {
3677 HasProfileData |= EI.second.Hotness != CalleeInfo::HotnessType::Unknown;
3687 HasProfileData |=
3688 EI.second.getHotness() != CalleeInfo::HotnessType::Unknown;
36783689 if (HasProfileData)
36793690 break;
36803691 }
268268 };
269269
270270 const auto NewThreshold =
271 Threshold * GetBonusMultiplier(Edge.second.Hotness);
271 Threshold * GetBonusMultiplier(Edge.second.getHotness());
272272
273273 auto *CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold,
274274 Summary.modulePath());
292292 return Threshold * ImportInstrFactor;
293293 };
294294
295 bool IsHotCallsite = Edge.second.Hotness == CalleeInfo::HotnessType::Hot;
295 bool IsHotCallsite =
296 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
296297 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
297298
298299 auto ExportModulePath = ResolvedCalleeSummary->modulePath();
305305 return nullptr;
306306 STRINGIFY_CODE(FS, PERMODULE)
307307 STRINGIFY_CODE(FS, PERMODULE_PROFILE)
308 STRINGIFY_CODE(FS, PERMODULE_RELBF)
308309 STRINGIFY_CODE(FS, PERMODULE_GLOBALVAR_INIT_REFS)
309310 STRINGIFY_CODE(FS, COMBINED)
310311 STRINGIFY_CODE(FS, COMBINED_PROFILE)