llvm.org GIT mirror llvm / c7d7fb0
[PGO] Differentiate Clang instrumentation and IR level instrumentation profiles This patch uses one bit in profile version to differentiate Clang instrumentation and IR level instrumentation profiles. PGOInstrumenation generates a COMDAT variable __llvm_profile_raw_version so that the compiler runtime can set the right profile kind. For Maco-O platform, we generate the variable as linkonce_odr linkage as COMDAT is not supported. PGOInstrumenation now checks this bit to make sure it's an IR level instrumentation profile. The patch was submitted as r260164 but reverted due to a Darwin test breakage. Original Differential Revision: http://reviews.llvm.org/D15540 Differential Revision: http://reviews.llvm.org/D17020 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@260385 91177308-0d34-0410-b5e6-96231b3b80d8 Rong Xu 3 years ago
25 changed file(s) with 156 addition(s) and 14 deletion(s). Raw diff Collapse all Expand all
704704 * version for other variants of profile. We set the lowest bit of the upper 8
705705 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
706706 * generated profile, and 0 if this is a Clang FE generated profile.
707 */
707 */
708708 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
709709 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
710 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
711 #define IR_LEVEL_PROF_VERSION_VAR __llvm_profile_raw_version
710712
711713 /* Runtime section names and name strings. */
712714 #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
6363 /// Iterator over profile data.
6464 InstrProfIterator begin() { return InstrProfIterator(this); }
6565 InstrProfIterator end() { return InstrProfIterator(); }
66 virtual bool isIRLevelProfile() const = 0;
6667
6768 /// Return the PGO symtab. There are three different readers:
6869 /// Raw, Text, and Indexed profile readers. The first two types
117118 std::unique_ptr DataBuffer;
118119 /// Iterator over the profile data.
119120 line_iterator Line;
121 bool IsIRLevelProfile;
120122
121123 TextInstrProfReader(const TextInstrProfReader &) = delete;
122124 TextInstrProfReader &operator=(const TextInstrProfReader &) = delete;
124126
125127 public:
126128 TextInstrProfReader(std::unique_ptr DataBuffer_)
127 : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {}
129 : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#'),
130 IsIRLevelProfile(false) {}
128131
129132 /// Return true if the given buffer is in text instrprof format.
130133 static bool hasFormat(const MemoryBuffer &Buffer);
134
135 bool isIRLevelProfile() const override { return IsIRLevelProfile; }
131136
132137 /// Read the header.
133138 std::error_code readHeader() override;
153158 /// The profile data file contents.
154159 std::unique_ptr DataBuffer;
155160 bool ShouldSwapBytes;
161 // The value of the version field of the raw profile data header. The lower 56
162 // bits specifies the format version and the most significant 8 bits specify
163 // the variant types of the profile.
164 uint64_t Version;
156165 uint64_t CountersDelta;
157166 uint64_t NamesDelta;
158167 const RawInstrProf::ProfileData *Data;
176185 static bool hasFormat(const MemoryBuffer &DataBuffer);
177186 std::error_code readHeader() override;
178187 std::error_code readNextRecord(InstrProfRecord &Record) override;
188 bool isIRLevelProfile() const override {
189 return (Version & VARIANT_MASK_IR_PROF) != 0;
190 }
179191
180192 InstrProfSymtab &getSymtab() override {
181193 assert(Symtab.get());
291303 virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
292304 virtual ~InstrProfReaderIndexBase() {}
293305 virtual uint64_t getVersion() const = 0;
306 virtual bool isIRLevelProfile() const = 0;
294307 virtual void populateSymtab(InstrProfSymtab &) = 0;
295308 };
296309
322335 HashTable->getInfoObj().setValueProfDataEndianness(Endianness);
323336 }
324337 ~InstrProfReaderIndex() override {}
325 uint64_t getVersion() const override { return FormatVersion; }
338 uint64_t getVersion() const override { return GET_VERSION(FormatVersion); }
339 bool isIRLevelProfile() const override {
340 return (FormatVersion & VARIANT_MASK_IR_PROF) != 0;
341 }
326342 void populateSymtab(InstrProfSymtab &Symtab) override {
327343 Symtab.create(HashTable->keys());
328344 }
347363 const unsigned char *Cur);
348364
349365 public:
366 /// Return the profile version.
350367 uint64_t getVersion() const { return Index->getVersion(); }
368 bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); }
351369 IndexedInstrProfReader(std::unique_ptr DataBuffer)
352370 : DataBuffer(std::move(DataBuffer)), Index(nullptr) {}
353371
2929 class InstrProfWriter {
3030 public:
3131 typedef SmallDenseMap ProfilingData;
32 enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel };
3233
3334 private:
3435 bool Sparse;
3536 StringMap FunctionData;
37 ProfKind ProfileKind;
3638 // Use raw pointer here for the incomplete type object.
3739 InstrProfRecordWriterTrait *InfoObj;
3840
5456 /// Write the profile, returning the raw data. For testing.
5557 std::unique_ptr writeBuffer();
5658
59 /// Set the ProfileKind. Report error if mixing FE and IR level profiles.
60 std::error_code setIsIRLevelProfile(bool IsIRLevel) {
61 if (ProfileKind == PF_Unknown) {
62 ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE;
63 return instrprof_error::success;
64 }
65 return (IsIRLevel == (ProfileKind == PF_IRLevel)) ?
66 instrprof_error::success : instrprof_error::unsupported_version;
67 }
68
5769 // Internal interface for testing purpose only.
5870 void setValueProfDataEndianness(support::endianness Endianness);
5971 void setOutputSparse(bool Sparse);
108108 [](char c) { return ::isprint(c) || ::isspace(c); });
109109 }
110110
111 // Read the profile variant flag from the header: ":FE" means this is a FE
112 // generated profile. ":IR" means this is an IR level profile. Other strings
113 // with a leading ':' will be reported an error format.
111114 std::error_code TextInstrProfReader::readHeader() {
112115 Symtab.reset(new InstrProfSymtab());
116 bool IsIRInstr = false;
117 if (!Line->startswith(":")) {
118 IsIRLevelProfile = false;
119 return success();
120 }
121 StringRef Str = (Line)->substr(1);
122 if (Str.equals_lower("ir"))
123 IsIRInstr = true;
124 else if (Str.equals_lower("fe"))
125 IsIRInstr = false;
126 else
127 return instrprof_error::bad_header;
128
129 ++Line;
130 IsIRLevelProfile = IsIRInstr;
113131 return success();
114132 }
115133
292310 template
293311 std::error_code
294312 RawInstrProfReader::readHeader(const RawInstrProf::Header &Header) {
295 if (swap(Header.Version) != RawInstrProf::Version)
313 Version = swap(Header.Version);
314 if (GET_VERSION(Version) != RawInstrProf::Version)
296315 return error(instrprof_error::unsupported_version);
297316
298317 CountersDelta = swap(Header.CountersDelta);
469488 return data_type();
470489 uint64_t Hash = endian::readNext(D);
471490
472 // Initialize number of counters for FormatVersion == 1.
491 // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
473492 uint64_t CountsSize = N / sizeof(uint64_t) - 1;
474493 // If format version is different then read the number of counters.
475 if (FormatVersion != IndexedInstrProf::ProfVersion::Version1) {
494 if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
476495 if (D + sizeof(uint64_t) > End)
477496 return data_type();
478497 CountsSize = endian::readNext(D);
489508 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
490509
491510 // Read value profiling data.
492 if (FormatVersion > IndexedInstrProf::ProfVersion::Version2 &&
511 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
493512 !readValueProfilingData(D, End)) {
494513 DataBuffer.clear();
495514 return data_type();
602621
603622 // Read the version.
604623 uint64_t FormatVersion = endian::byte_swap(Header->Version);
605 if (FormatVersion > IndexedInstrProf::ProfVersion::CurrentVersion)
624 if (GET_VERSION(FormatVersion) >
625 IndexedInstrProf::ProfVersion::CurrentVersion)
606626 return error(instrprof_error::unsupported_version);
607627
608628 Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur);
141141 }
142142
143143 InstrProfWriter::InstrProfWriter(bool Sparse)
144 : Sparse(Sparse), FunctionData(),
144 : Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown),
145145 InfoObj(new InstrProfRecordWriterTrait()) {}
146146
147147 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
229229 IndexedInstrProf::Header Header;
230230 Header.Magic = IndexedInstrProf::Magic;
231231 Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
232 if (ProfileKind == PF_IRLevel)
233 Header.Version |= VARIANT_MASK_IR_PROF;
232234 Header.Unused = 0;
233235 Header.HashType = static_cast(IndexedInstrProf::HashType);
234236 Header.HashOffset = 0;
335337 }
336338
337339 void InstrProfWriter::writeText(raw_fd_ostream &OS) {
340 if (ProfileKind == PF_IRLevel)
341 OS << "# IR level Instrumentation Flag\n:ir\n";
338342 InstrProfSymtab Symtab;
339343 for (const auto &I : FunctionData)
340344 if (shouldEncodeData(I.getValue()))
4848 #include "llvm/ADT/DenseMap.h"
4949 #include "llvm/ADT/STLExtras.h"
5050 #include "llvm/ADT/Statistic.h"
51 #include "llvm/ADT/Triple.h"
5152 #include "llvm/Analysis/BlockFrequencyInfo.h"
5253 #include "llvm/Analysis/BranchProbabilityInfo.h"
5354 #include "llvm/Analysis/CFG.h"
712713 }
713714 } // end anonymous namespace
714715
716 // Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime
717 // aware this is an ir_level profile so it can set the version flag.
718 static void createIRLevelProfileFlagVariable(Module &M) {
719 Type *IntTy64 = Type::getInt64Ty(M.getContext());
720 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
721 auto IRLevelVersionVariable =
722 new GlobalVariable(M, IntTy64, true, GlobalVariable::ExternalLinkage,
723 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
724 INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR));
725 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
726 Triple TT(M.getTargetTriple());
727 if (TT.isOSBinFormatMachO())
728 IRLevelVersionVariable->setLinkage(GlobalValue::LinkOnceODRLinkage);
729 else
730 IRLevelVersionVariable->setComdat(
731 M.getOrInsertComdat(StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR))));
732 }
733
715734 bool PGOInstrumentationGen::runOnModule(Module &M) {
735 createIRLevelProfileFlagVariable(M);
716736 for (auto &F : M) {
717737 if (F.isDeclaration())
718738 continue;
750770 "Cannot get PGOReader"));
751771 return false;
752772 }
773 // TODO: might need to change the warning once the clang option is finalized.
774 if (!PGOReader->isIRLevelProfile()) {
775 Ctx.diagnose(DiagnosticInfoPGOProfile(
776 ProfileFileName.data(), "Not an IR level instrumentation profile"));
777 return false;
778 }
779
753780
754781 for (auto &F : M) {
755782 if (F.isDeclaration())
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_br_1
13 25571299074
24 2
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_br_2
13 29667547796
24 2
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_criticalEdge
13 82323253069
24 8
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 foo
13 12884999999
24 1
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 foo
13 59130013419
24 4
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_simple_for
13 34137660316
24 2
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_nested_for
13 53929068288
24 3
0 # :ir is the flag to indicate this is IR level profile.
1 :ir
2 test_switch
13 46200943743
24 4
None ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
0 ; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-COMDAT
1 ; RUN: opt < %s -mtriple=x86_64-apple-darwin -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-DARWIN-LINKONCE
12 ; RUN: llvm-profdata merge %S/Inputs/branch1.proftext -o %t.profdata
23 ; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
34 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
45 target triple = "x86_64-unknown-linux-gnu"
6 ; GEN-DARWIN-LINKONCE: target triple = "x86_64-apple-darwin"
57
8 ; GEN-COMDAT: $__llvm_profile_raw_version = comdat any
9 ; GEN-COMDAT: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
10 ; GEN-LINKONCE: @__llvm_profile_raw_version = linkonce constant i64 72057594037927939
611 ; GEN: @__profn_test_br_1 = private constant [9 x i8] c"test_br_1"
712
813 define i32 @test_br_1(i32 %i) {
33 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
44 target triple = "x86_64-unknown-linux-gnu"
55
6 ; GEN: $__llvm_profile_raw_version = comdat any
7 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
68 ; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2"
79
810 define i32 @test_br_2(i32 %i) {
33 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
44 target triple = "x86_64-unknown-linux-gnu"
55
6 ; GEN: $__llvm_profile_raw_version = comdat any
7 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
68 ; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge"
79 ; GEN: @__profn__stdin__bar = private constant [11 x i8] c":bar"
810
0 ; RUN: llvm-profdata merge %S/Inputs/diag_FE.proftext -o %t.profdata
1 ; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
2
3 ; CHECK: Not an IR level instrumentation profile
4
5 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
6 target triple = "x86_64-unknown-linux-gnu"
7
8 define i32 @foo() {
9 entry:
10 ret i32 0
11 }
55
66 @val = global i32 0, align 4
77 @_ZTIi = external constant i8*
8 ; GEN: $__llvm_profile_raw_version = comdat any
9 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
810 ; GEN: @__profn_bar = private constant [3 x i8] c"bar"
911 ; GEN: @__profn_foo = private constant [3 x i8] c"foo"
1012
33 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
44 target triple = "x86_64-unknown-linux-gnu"
55
6 ; GEN: $__llvm_profile_raw_version = comdat any
7 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
68 ; GEN: @__profn_test_simple_for = private constant [15 x i8] c"test_simple_for"
79
810 define i32 @test_simple_for(i32 %n) {
33 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
44 target triple = "x86_64-unknown-linux-gnu"
55
6 ; GEN: $__llvm_profile_raw_version = comdat any
7 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
68 ; GEN: @__profn_test_nested_for = private constant [15 x i8] c"test_nested_for"
79
810 define i32 @test_nested_for(i32 %r, i32 %s) {
11 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
22 target triple = "x86_64-unknown-linux-gnu"
33
4 ; GEN: $__llvm_profile_raw_version = comdat any
5 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
46 ; GEN: @__profn_single_bb = private constant [9 x i8] c"single_bb"
57
68 define i32 @single_bb() {
33 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
44 target triple = "x86_64-unknown-linux-gnu"
55
6 ; GEN: $__llvm_profile_raw_version = comdat any
7 ; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
68 ; GEN: @__profn_test_switch = private constant [11 x i8] c"test_switch"
79
810 define void @test_switch(i32 %i) {
127127 exitWithErrorCode(ec, Input.Filename);
128128
129129 auto Reader = std::move(ReaderOrErr.get());
130 bool IsIRProfile = Reader->isIRLevelProfile();
131 if (Writer.setIsIRLevelProfile(IsIRProfile))
132 exitWithError("Merge IR generated profile with Clang generated profile.");
133
130134 for (auto &I : *Reader) {
131135 if (std::error_code EC = Writer.addRecord(std::move(I), Input.Weight)) {
132136 // Only show hint the first time an error occurs.
268272 exitWithErrorCode(EC, Filename);
269273
270274 auto Reader = std::move(ReaderOrErr.get());
275 bool IsIRInstr = Reader->isIRLevelProfile();
271276 size_t ShownFunctions = 0;
272277 for (const auto &Func : *Reader) {
273278 bool Show =
294299
295300 OS << " " << Func.Name << ":\n"
296301 << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n"
297 << " Counters: " << Func.Counts.size() << "\n"
298 << " Function count: " << Func.Counts[0] << "\n";
302 << " Counters: " << Func.Counts.size() << "\n";
303 if (!IsIRInstr)
304 OS << " Function count: " << Func.Counts[0] << "\n";
299305
300306 if (ShowIndirectCallTargets)
301307 OS << " Indirect Call Site Count: "
303309
304310 if (ShowCounts) {
305311 OS << " Block counts: [";
306 for (size_t I = 1, E = Func.Counts.size(); I < E; ++I) {
307 OS << (I == 1 ? "" : ", ") << Func.Counts[I];
312 size_t Start = (IsIRInstr ? 0 : 1);
313 for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) {
314 OS << (I == Start ? "" : ", ") << Func.Counts[I];
308315 }
309316 OS << "]\n";
310317 }