20#include "llvm/Config/config.h"
54#include <system_error>
61#define DEBUG_TYPE "instrprof"
65 cl::desc(
"Use full module build paths in the profile counter names for "
66 "static functions."));
78 cl::desc(
"Strip specified level of directory name from source path in "
79 "the profile counter name for static functions."));
82 const std::string &ErrMsg =
"") {
87 case instrprof_error::success:
90 case instrprof_error::eof:
93 case instrprof_error::unrecognized_format:
94 OS <<
"unrecognized instrumentation profile encoding format";
96 case instrprof_error::bad_magic:
97 OS <<
"invalid instrumentation profile data (bad magic)";
99 case instrprof_error::bad_header:
100 OS <<
"invalid instrumentation profile data (file header is corrupt)";
102 case instrprof_error::unsupported_version:
103 OS <<
"unsupported instrumentation profile format version";
105 case instrprof_error::unsupported_hash_type:
106 OS <<
"unsupported instrumentation profile hash type";
108 case instrprof_error::too_large:
109 OS <<
"too much profile data";
111 case instrprof_error::truncated:
112 OS <<
"truncated profile data";
114 case instrprof_error::malformed:
115 OS <<
"malformed instrumentation profile data";
117 case instrprof_error::missing_correlation_info:
118 OS <<
"debug info/binary for correlation is required";
120 case instrprof_error::unexpected_correlation_info:
121 OS <<
"debug info/binary for correlation is not necessary";
123 case instrprof_error::unable_to_correlate_profile:
124 OS <<
"unable to correlate profile";
126 case instrprof_error::invalid_prof:
127 OS <<
"invalid profile created. Please file a bug "
128 "at: " BUG_REPORT_URL
129 " and include the profraw files that caused this error.";
131 case instrprof_error::unknown_function:
132 OS <<
"no profile data available for function";
134 case instrprof_error::hash_mismatch:
135 OS <<
"function control flow change detected (hash mismatch)";
137 case instrprof_error::count_mismatch:
138 OS <<
"function basic block count change detected (counter mismatch)";
140 case instrprof_error::bitmap_mismatch:
141 OS <<
"function bitmap size change detected (bitmap size mismatch)";
143 case instrprof_error::counter_overflow:
144 OS <<
"counter overflow";
146 case instrprof_error::value_site_count_mismatch:
147 OS <<
"function value site count change detected (counter mismatch)";
149 case instrprof_error::compress_failed:
150 OS <<
"failed to compress data (zlib)";
152 case instrprof_error::uncompress_failed:
153 OS <<
"failed to uncompress data (zlib)";
155 case instrprof_error::empty_raw_profile:
156 OS <<
"empty raw profile file";
158 case instrprof_error::zlib_unavailable:
159 OS <<
"profile uses zlib compression but the profile reader was built "
160 "without zlib support";
162 case instrprof_error::raw_profile_version_mismatch:
163 OS <<
"raw profile version mismatch";
165 case instrprof_error::counter_value_too_large:
166 OS <<
"excessively large counter value suggests corrupted profile data";
172 OS <<
": " << ErrMsg;
182class InstrProfErrorCategoryType :
public std::error_category {
183 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
185 std::string message(
int IE)
const override {
193 static InstrProfErrorCategoryType ErrorCategory;
194 return ErrorCategory;
199const char *InstrProfSectNameCommon[] = {
200#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
205const char *InstrProfSectNameCoff[] = {
206#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
211const char *InstrProfSectNamePrefix[] = {
212#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
222 "enable-name-compression",
226 "enable-vtable-value-profiling",
cl::init(
false),
227 cl::desc(
"If true, the virtual table address will be instrumented to know "
228 "the types of a C++ pointer. The information is used in indirect "
229 "call promotion to do selective vtable-based comparison."));
233 bool AddSegmentInfo) {
234 std::string SectName;
237 SectName = InstrProfSectNamePrefix[IPSK];
240 SectName += InstrProfSectNameCoff[IPSK];
242 SectName += InstrProfSectNameCommon[IPSK];
244 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
245 SectName +=
",regular,live_support";
265 std::string NewName = std::string(
Name);
271 if (FileName.
empty())
272 NewName = NewName.insert(0,
"<unknown>:");
274 NewName = NewName.insert(0, FileName.
str() +
":");
285 for (
auto & CI : PathNameStr) {
294 return PathNameStr.
substr(LastPos);
345 MDNode *PGONameMetadata) {
353 return *IRPGOFuncName;
400 if (MangledName.empty())
401 return std::make_pair(
StringRef(), IRPGOName);
402 return std::make_pair(FileName, MangledName);
406 if (FileName.
empty())
426 const char InvalidChars[] =
"-:;<>/\"'";
427 size_t found = VarName.find_first_of(InvalidChars);
428 while (found != std::string::npos) {
429 VarName[found] =
'_';
430 found = VarName.find_first_of(InvalidChars, found + 1);
481 if (!
G.hasName() || !
G.hasMetadata(LLVMContext::MD_type))
483 if (
Error E = addVTableWithName(
499 bool Inserted =
true;
500 std::tie(std::ignore, Inserted) =
506 if (
Error E = mapName(VTablePGOName))
509 StringRef CanonicalName = getCanonicalName(VTablePGOName);
510 if (CanonicalName != VTablePGOName)
511 return mapName(CanonicalName);
523 const uint8_t *EndP = NameStrings.
bytes_end();
530 bool isCompressed = (CompressedSize != 0);
538 UncompressedNameStrings,
544 NameStrings =
toStringRef(UncompressedNameStrings);
547 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
548 P += UncompressedSize;
557 while (
P < EndP && *
P == 0)
573 this, std::placeholders::_1)))
584 CompressedVTableStrings,
598 const std::string UniqSuffix =
".__uniq.";
599 size_t pos = PGOName.
find(UniqSuffix);
601 pos += UniqSuffix.length();
607 pos = PGOName.
find(
'.', pos);
609 return PGOName.
substr(0, pos);
621 if (
Error E = mapName(PGOFuncName))
624 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
625 if (CanonicalFuncName != PGOFuncName)
626 return mapName(CanonicalFuncName);
634 return VTableAddrMap.
lookup(Address, 0);
639 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
640 return A.first < Address;
646 if (It != AddrToMD5Map.end() && It->first == Address)
659 bool doCompression, std::string &Result) {
660 assert(!NameStrs.
empty() &&
"No name data to emit");
662 uint8_t Header[20], *
P = Header;
663 std::string UncompressedNameStrings =
668 "PGO name is invalid (contains separator token)");
670 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
673 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
676 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
677 unsigned HeaderLen =
P - &Header[0];
678 Result.append(HeaderStr, HeaderLen);
683 if (!doCompression) {
684 return WriteStringToResult(0, UncompressedNameStrings);
689 CompressedNameStrings,
692 return WriteStringToResult(CompressedNameStrings.
size(),
699 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
704 std::string &Result,
bool doCompression) {
705 std::vector<std::string> NameStrs;
706 for (
auto *NameVar : NameVars) {
714 std::string &Result,
bool doCompression) {
715 std::vector<std::string> VTableNameStrs;
716 for (
auto *VTable : VTables)
717 VTableNameStrs.push_back(
getPGOName(*VTable));
730 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
733 for (
size_t I = 0;
I < NumValueSites; ++
I) {
737 KindSum += VD[V].Count;
749 double Score = 0.0f, FuncLevelScore = 0.0f;
754 while (
I != IE && J != JE) {
755 if (
I->Value == J->Value) {
763 }
else if (
I->Value < J->Value) {
779 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
780 if (!ThisNumValueSites)
783 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
784 getOrCreateValueSitesForKind(ValueKind);
786 Other.getValueSitesForKind(ValueKind);
788 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
798 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
802 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
805 if (ThisNumValueSites != OtherNumValueSites) {
817 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
823 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
826 MaxCount = std::max(
Other.Counts[
I], MaxCount);
831 if (MaxCount >= ValueCutoff) {
832 double FuncScore = 0.0;
833 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I)
839 FuncLevelOverlap.
Valid =
true;
850 for (
const InstrProfValueData &J : Input.
ValueData) {
851 while (
I != IE &&
I->Value < J.Value)
853 if (
I != IE &&
I->Value == J.Value) {
877void InstrProfRecord::mergeValueProfData(
881 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
882 if (ThisNumValueSites != OtherNumValueSites) {
886 if (!ThisNumValueSites)
888 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
889 getOrCreateValueSitesForKind(ValueKind);
891 Src.getValueSitesForKind(ValueKind);
893 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
923 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
944 for (
size_t I = 0, E =
Other.BitmapBytes.size();
I < E; ++
I) {
948 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
949 mergeValueProfData(Kind,
Other, Weight, Warn);
952void InstrProfRecord::scaleValueProfData(
955 for (
auto &R : getValueSitesForKind(ValueKind))
961 assert(
D != 0 &&
"D cannot be 0");
962 for (
auto &Count : this->
Counts) {
972 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
973 scaleValueProfData(Kind,
N,
D, Warn);
982 if (ValueKind == IPVK_IndirectCallTarget)
985 if (ValueKind == IPVK_VTableTarget)
997 std::vector<InstrProfValueSiteRecord> &ValueSites =
998 getOrCreateValueSitesForKind(ValueKind);
1000 ValueSites.emplace_back();
1002 ValueSites.emplace_back(VData, VData +
N);
1012 size_t LargestTraceSize = 0;
1013 for (
auto &
Trace : Traces)
1015 std::max(LargestTraceSize,
Trace.FunctionNameRefs.
size());
1017 for (
auto &
Trace : Traces)
1021 const int N =
Log2_64(LargestTraceSize) + 1;
1026 for (
size_t TraceIdx = 0; TraceIdx < Traces.
size(); TraceIdx++) {
1027 auto &
Trace = Traces[TraceIdx].FunctionNameRefs;
1031 UtilityNodeT GroupId = TraceIdx *
N +
I;
1037 std::vector<BPFunctionNode> Nodes;
1038 for (
auto Id : FunctionIds) {
1039 auto &UNs = FuncGroups[Id];
1041 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end());
1042 Nodes.emplace_back(Id, UNs);
1047#define INSTR_PROF_COMMON_API_IMPL
1061 ->getNumValueSites(VKind);
1066 ->getNumValueData(VKind);
1072 ->getNumValueDataForSite(VK, S);
1077 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
1082 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
1083 memset(VD, 0, TotalSizeInBytes);
1100 Closure.Record = &
Record;
1101 return getValueProfDataSize(&Closure);
1105std::unique_ptr<ValueProfData>
1109 std::unique_ptr<ValueProfData> VPD(
1116 Record.reserveSites(Kind, NumValueSites);
1118 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
1119 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1120 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1121 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1122 ValueData += ValueDataCount;
1130 using namespace support;
1136 sys::swapByteOrder<uint32_t>(NumValueSites);
1137 sys::swapByteOrder<uint32_t>(Kind);
1139 uint32_t ND = getValueProfRecordNumValueData(
this);
1140 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1144 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1145 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1148 sys::swapByteOrder<uint32_t>(NumValueSites);
1149 sys::swapByteOrder<uint32_t>(Kind);
1155 if (NumValueKinds == 0)
1158 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1159 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1160 VR->deserializeTo(
Record, SymTab);
1161 VR = getValueProfRecordNext(VR);
1167 using namespace support;
1170 return endian::readNext<T, llvm::endianness::little>(
D);
1172 return endian::readNext<T, llvm::endianness::big>(
D);
1176 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1180Error ValueProfData::checkIntegrity() {
1181 if (NumValueKinds > IPVK_Last + 1)
1182 return make_error<InstrProfError>(
1186 return make_error<InstrProfError>(
1189 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1190 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1191 if (VR->Kind > IPVK_Last)
1193 "value kind is invalid");
1194 VR = getValueProfRecordNext(VR);
1195 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1196 return make_error<InstrProfError>(
1198 "value profile address is greater than total size");
1204ValueProfData::getValueProfData(
const unsigned char *
D,
1205 const unsigned char *
const BufferEnd,
1207 using namespace support;
1209 if (
D +
sizeof(ValueProfData) > BufferEnd)
1212 const unsigned char *Header =
D;
1213 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
1214 if (
D + TotalSize > BufferEnd)
1218 memcpy(VPD.get(),
D, TotalSize);
1220 VPD->swapBytesToHost(Endianness);
1222 Error E = VPD->checkIntegrity();
1224 return std::move(E);
1226 return std::move(VPD);
1230 using namespace support;
1235 sys::swapByteOrder<uint32_t>(TotalSize);
1236 sys::swapByteOrder<uint32_t>(NumValueKinds);
1238 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1239 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1241 VR = getValueProfRecordNext(VR);
1246 using namespace support;
1251 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1252 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1253 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1257 sys::swapByteOrder<uint32_t>(TotalSize);
1258 sys::swapByteOrder<uint32_t>(NumValueKinds);
1270 std::unique_ptr<InstrProfValueData[]> VD =
1297 for (
auto &VD : VDs) {
1318 if (!
Tag ||
Tag->getString() !=
"VP")
1333 InstrProfValueData ValueData[],
1335 uint64_t &TotalC,
bool GetNoICPValue) {
1342 ActualNumValueData = 0;
1344 for (
unsigned I = 3;
I < NOps;
I += 2) {
1345 if (ActualNumValueData >= MaxNumDataWant)
1349 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1350 if (!
Value || !Count)
1355 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1356 ValueData[ActualNumValueData].Count = CntValue;
1357 ActualNumValueData++;
1362std::unique_ptr<InstrProfValueData[]>
1365 uint64_t &TotalC,
bool GetNoICPValue) {
1369 auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
1371 ActualNumValueData, TotalC, GetNoICPValue))
1373 return ValueDataArray;
1381 InstrProfValueData ValueData[],
1383 bool GetNoICPValue) {
1388 ActualNumValueData, TotalC,
1398 if (PGOFuncName ==
F.getName())
1436 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1437 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1442 if (IRInstrVar->isDeclaration())
1446 if (!IRInstrVar->hasInitializer())
1449 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1452 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1457 if (
F.getName().empty())
1463 if (CheckAddressTaken &&
F.hasAddressTaken())
1471 if (!
F.hasComdat()) {
1480 if (InstrProfileOutput.
empty())
1486 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1488 Triple TT(M.getTargetTriple());
1489 if (TT.supportsCOMDAT()) {
1491 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1492 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1497 const std::string &TestFilename,
1499 auto getProfileSum = [IsCS](
const std::string &Filename,
1505 if (
Error E = ReaderOrErr.takeError()) {
1508 auto Reader = std::move(ReaderOrErr.get());
1509 Reader->accumulateCounts(Sum, IsCS);
1527 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1537 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1547 const char *EntryName =
1548 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1550 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1551 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1553 OS <<
"Function level:\n"
1562 OS <<
" # of " << EntryName
1568 OS <<
" Mismatched count percentage (Edge): "
1571 OS <<
" Percentage of Edge profile only in test_profile: "
1578 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1581 char ProfileKindName[20] = {0};
1583 case IPVK_IndirectCallTarget:
1584 strncpy(ProfileKindName,
"IndirectCall", 19);
1586 case IPVK_MemOPSize:
1587 strncpy(ProfileKindName,
"MemOP", 19);
1589 case IPVK_VTableTarget:
1590 strncpy(ProfileKindName,
"VTable", 19);
1593 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1596 OS <<
" " << ProfileKindName
1600 OS <<
" Mismatched count percentage (" << ProfileKindName
1603 OS <<
" Percentage of " << ProfileKindName
1604 <<
" profile only in test_profile: "
1606 OS <<
" " << ProfileKindName
1609 <<
" " << ProfileKindName
1615namespace IndexedInstrProf {
1617template <
typename T1,
typename T2>
1619 constexpr T2 Object{};
1620 return size_t(&(Object.*Member)) - size_t(&Object);
1628 using namespace support;
1629 return endian::byte_swap<uint64_t, llvm::endianness::little>(
Version);
1633 using namespace support;
1634 static_assert(std::is_standard_layout_v<Header>,
1635 "The header should be standard layout type since we use offset "
1636 "of fields to read.");
1642 endian::byte_swap<uint64_t, llvm::endianness::little>(
H.Magic);
1648 if (GET_VERSION(
H.formatVersion()) >
1652 switch (GET_VERSION(
H.formatVersion())) {
1657 "Please update the reading code below if a new field has been added, "
1658 "if not add a case statement to fall through to the latest version.");
1665 H.TemporalProfTracesOffset =
1688 "Please update the size computation below if a new field has "
1689 "been added to the header, if not add a case statement to "
1690 "fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Error addSymbolName(StringRef SymbolName)
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
ValT lookup(KeyT x, ValT NotFound=ValT()) const
lookup - Return the mapped value at x or NotFound.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
const std::string & getSourceFileName() const
Get the module's original source file name.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
A vector that has set insertion semantics.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
static constexpr size_t npos
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This class represents a function that is read from a sample profile.
@ C
The default llvm calling convention, compatible with C.
static uint64_t read(const unsigned char *Buffer, size_t Offset)
size_t constexpr offsetOf(T1 T2::*Member)
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
constexpr char kGlobalIdentifierDelimiter
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig)
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
static bool getValueProfDataFromInstImpl(const MDNode *const MD, const uint32_t MaxNumDataWant, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue)
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
double ValueCounts[IPVK_Last - IPVK_First+1]
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::vector< uint8_t > BitmapBytes
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...