llvm.org GIT mirror llvm / ec13dea
[SampleFDO] Add FunctionOffsetTable in compact binary format profile. The patch saves a function offset table which maps function name index to the offset of its function profile to the start of the binary profile. By using the function offset table, for those function profiles which will not be used when compiling a module, the profile reader does't have to read them. For profile size around 10~20M, it saves ~10% compile time. Differential Revision: https://reviews.llvm.org/D51863 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@342283 91177308-0d34-0410-b5e6-96231b3b80d8 Wei Mi 1 year, 29 days ago
11 changed file(s) with 250 addition(s) and 29 deletion(s). Raw diff Collapse all Expand all
4848 unsupported_writing_format,
4949 truncated_name_table,
5050 not_implemented,
51 counter_overflow
51 counter_overflow,
52 ostream_seek_unsupported
5253 };
5354
5455 inline std::error_code make_error_code(sampleprof_error E) {
278278 /// Print the profile for \p FName on stream \p OS.
279279 void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs());
280280
281 virtual void collectFuncsToUse(const Module &M) {}
282
281283 /// Print all the profiles on stream \p OS.
282284 void dump(raw_ostream &OS = dbgs());
283285
363365 : SampleProfileReader(std::move(B), C, Format) {}
364366
365367 /// Read and validate the file header.
366 std::error_code readHeader() override;
368 virtual std::error_code readHeader() override;
367369
368370 /// Read sample profiles from the associated file.
369371 std::error_code read() override;
377379 /// \returns the read value.
378380 template ErrorOr readNumber();
379381
382 /// Read a numeric value of type T from the profile. The value is saved
383 /// without encoded.
384 template ErrorOr readUnencodedNumber();
385
380386 /// Read a string from the profile.
381387 ///
382388 /// If an error occurs during decoding, a diagnostic message is emitted and
390396
391397 /// Return true if we've reached the end of file.
392398 bool at_eof() const { return Data >= End; }
399
400 /// Read the next function profile instance.
401 std::error_code readFuncProfile();
393402
394403 /// Read the contents of the given profile instance.
395404 std::error_code readProfile(FunctionSamples &FProfile);
435444 private:
436445 /// Function name table.
437446 std::vector NameTable;
447 /// The table mapping from function name to the offset of its FunctionSample
448 /// towards file start.
449 DenseMap FuncOffsetTable;
450 /// The set containing the functions to use when compiling a module.
451 DenseSet FuncsToUse;
438452 virtual std::error_code verifySPMagic(uint64_t Magic) override;
439453 virtual std::error_code readNameTable() override;
440454 /// Read a string indirectly via the name table.
441455 virtual ErrorOr readStringFromTable() override;
456 virtual std::error_code readHeader() override;
457 std::error_code readFuncOffsetTable();
442458
443459 public:
444460 SampleProfileReaderCompactBinary(std::unique_ptr B,
447463
448464 /// \brief Return true if \p Buffer is in the format supported by this class.
449465 static bool hasFormat(const MemoryBuffer &Buffer);
466
467 /// Read samples only for functions to use.
468 std::error_code read() override;
469
470 /// Collect functions to be used when compiling Module \p M.
471 void collectFuncsToUse(const Module &M) override;
450472 };
451473
452474 using InlineCallStack = SmallVector;
4141 /// Write all the sample profiles in the given map of samples.
4242 ///
4343 /// \returns status code of the file update operation.
44 std::error_code write(const StringMap &ProfileMap);
44 virtual std::error_code write(const StringMap &ProfileMap);
4545
4646 raw_ostream &getOutputStream() { return *OutputStream; }
4747
102102 /// Sample-based profile writer (binary format).
103103 class SampleProfileWriterBinary : public SampleProfileWriter {
104104 public:
105 std::error_code write(const FunctionSamples &S) override;
105 virtual std::error_code write(const FunctionSamples &S) override;
106106 SampleProfileWriterBinary(std::unique_ptr &OS)
107107 : SampleProfileWriter(OS) {}
108108
109109 protected:
110110 virtual std::error_code writeNameTable() = 0;
111111 virtual std::error_code writeMagicIdent() = 0;
112 std::error_code writeHeader(const StringMap &ProfileMap) override;
112 virtual std::error_code
113 writeHeader(const StringMap &ProfileMap) override;
113114 std::error_code writeSummary();
114115 std::error_code writeNameIdx(StringRef FName);
115116 std::error_code writeBody(const FunctionSamples &S);
134135 virtual std::error_code writeMagicIdent() override;
135136 };
136137
138 // CompactBinary is a compact format of binary profile which both reduces
139 // the profile size and the load time needed when compiling. It has two
140 // major difference with Binary format.
141 // 1. It represents all the strings in name table using md5 hash.
142 // 2. It saves a function offset table which maps function name index to
143 // the offset of its function profile to the start of the binary profile,
144 // so by using the function offset table, for those function profiles which
145 // will not be needed when compiling a module, the profile reader does't
146 // have to read them and it saves compile time if the profile size is huge.
147 // The layout of the compact format is shown as follows:
148 //
149 // Part1: Profile header, the same as binary format, containing magic
150 // number, version, summary, name table...
151 // Part2: Function Offset Table Offset, which saves the position of
152 // Part4.
153 // Part3: Function profile collection
154 // function1 profile start
155 // ....
156 // function2 profile start
157 // ....
158 // function3 profile start
159 // ....
160 // ......
161 // Part4: Function Offset Table
162 // function1 name index --> function1 profile start
163 // function2 name index --> function2 profile start
164 // function3 name index --> function3 profile start
165 //
166 // We need Part2 because profile reader can use it to find out and read
167 // function offset table without reading Part3 first.
137168 class SampleProfileWriterCompactBinary : public SampleProfileWriterBinary {
138169 using SampleProfileWriterBinary::SampleProfileWriterBinary;
139170
171 public:
172 virtual std::error_code write(const FunctionSamples &S) override;
173 virtual std::error_code
174 write(const StringMap &ProfileMap) override;
175
140176 protected:
177 /// The table mapping from function name to the offset of its FunctionSample
178 /// towards profile start.
179 MapVector FuncOffsetTable;
180 /// The offset of the slot to be filled with the offset of FuncOffsetTable
181 /// towards profile start.
182 uint64_t TableOffset;
141183 virtual std::error_code writeNameTable() override;
142184 virtual std::error_code writeMagicIdent() override;
185 virtual std::error_code
186 writeHeader(const StringMap &ProfileMap) override;
187 std::error_code writeFuncOffsetTable();
143188 };
144189
145190 } // end namespace sampleprof
6666 return "Unimplemented feature";
6767 case sampleprof_error::counter_overflow:
6868 return "Counter overflow";
69 case sampleprof_error::ostream_seek_unsupported:
70 return "Ostream does not support seek";
6971 }
7072 llvm_unreachable("A value of sampleprof_error has no message.");
7173 }
2929 #include "llvm/Support/ErrorOr.h"
3030 #include "llvm/Support/LEB128.h"
3131 #include "llvm/Support/LineIterator.h"
32 #include "llvm/Support/MD5.h"
3233 #include "llvm/Support/MemoryBuffer.h"
3334 #include "llvm/Support/raw_ostream.h"
3435 #include
319320 }
320321
321322 template
323 ErrorOr SampleProfileReaderBinary::readUnencodedNumber() {
324 std::error_code EC;
325
326 if (Data + sizeof(T) > End) {
327 EC = sampleprof_error::truncated;
328 reportError(0, EC.message());
329 return EC;
330 }
331
332 using namespace support;
333 T Val = endian::readNext(Data);
334 return Val;
335 }
336
337 template
322338 inline ErrorOr SampleProfileReaderBinary::readStringIndex(T &Table) {
323339 std::error_code EC;
324340 auto Idx = readNumber();
422438 return sampleprof_error::success;
423439 }
424440
441 std::error_code SampleProfileReaderBinary::readFuncProfile() {
442 auto NumHeadSamples = readNumber();
443 if (std::error_code EC = NumHeadSamples.getError())
444 return EC;
445
446 auto FName(readStringFromTable());
447 if (std::error_code EC = FName.getError())
448 return EC;
449
450 Profiles[*FName] = FunctionSamples();
451 FunctionSamples &FProfile = Profiles[*FName];
452 FProfile.setName(*FName);
453
454 FProfile.addHeadSamples(*NumHeadSamples);
455
456 if (std::error_code EC = readProfile(FProfile))
457 return EC;
458 return sampleprof_error::success;
459 }
460
425461 std::error_code SampleProfileReaderBinary::read() {
426462 while (!at_eof()) {
427 auto NumHeadSamples = readNumber();
428 if (std::error_code EC = NumHeadSamples.getError())
429 return EC;
430
431 auto FName(readStringFromTable());
432 if (std::error_code EC = FName.getError())
433 return EC;
434
435 Profiles[*FName] = FunctionSamples();
436 FunctionSamples &FProfile = Profiles[*FName];
437 FProfile.setName(*FName);
438
439 FProfile.addHeadSamples(*NumHeadSamples);
440
441 if (std::error_code EC = readProfile(FProfile))
442 return EC;
443 }
444
463 if (std::error_code EC = readFuncProfile())
464 return EC;
465 }
466
467 return sampleprof_error::success;
468 }
469
470 std::error_code SampleProfileReaderCompactBinary::read() {
471 for (auto Name : FuncsToUse) {
472 auto GUID = std::to_string(MD5Hash(Name));
473 auto iter = FuncOffsetTable.find(StringRef(GUID));
474 if (iter == FuncOffsetTable.end())
475 continue;
476 const uint8_t *SavedData = Data;
477 Data = reinterpret_cast(Buffer->getBufferStart()) +
478 iter->second;
479 if (std::error_code EC = readFuncProfile())
480 return EC;
481 Data = SavedData;
482 }
445483 return sampleprof_error::success;
446484 }
447485
511549 if (std::error_code EC = readNameTable())
512550 return EC;
513551 return sampleprof_error::success;
552 }
553
554 std::error_code SampleProfileReaderCompactBinary::readHeader() {
555 SampleProfileReaderBinary::readHeader();
556 if (std::error_code EC = readFuncOffsetTable())
557 return EC;
558 return sampleprof_error::success;
559 }
560
561 std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
562 auto TableOffset = readUnencodedNumber();
563 if (std::error_code EC = TableOffset.getError())
564 return EC;
565
566 const uint8_t *SavedData = Data;
567 const uint8_t *TableStart =
568 reinterpret_cast(Buffer->getBufferStart()) +
569 *TableOffset;
570 Data = TableStart;
571
572 auto Size = readNumber();
573 if (std::error_code EC = Size.getError())
574 return EC;
575
576 FuncOffsetTable.reserve(*Size);
577 for (uint32_t I = 0; I < *Size; ++I) {
578 auto FName(readStringFromTable());
579 if (std::error_code EC = FName.getError())
580 return EC;
581
582 auto Offset = readNumber();
583 if (std::error_code EC = Offset.getError())
584 return EC;
585
586 FuncOffsetTable[*FName] = *Offset;
587 }
588 End = TableStart;
589 Data = SavedData;
590 return sampleprof_error::success;
591 }
592
593 void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) {
594 FuncsToUse.clear();
595 for (auto &F : M) {
596 StringRef Fname = F.getName().split('.').first;
597 FuncsToUse.insert(Fname);
598 }
514599 }
515600
516601 std::error_code SampleProfileReaderBinary::readSummaryEntry(
2121 #include "llvm/ADT/StringRef.h"
2222 #include "llvm/ProfileData/ProfileCommon.h"
2323 #include "llvm/ProfileData/SampleProf.h"
24 #include "llvm/Support/Endian.h"
25 #include "llvm/Support/EndianStream.h"
2426 #include "llvm/Support/ErrorOr.h"
2527 #include "llvm/Support/FileSystem.h"
2628 #include "llvm/Support/LEB128.h"
6062 if (std::error_code EC = write(*I.second))
6163 return EC;
6264 }
65 return sampleprof_error::success;
66 }
67
68 std::error_code SampleProfileWriterCompactBinary::write(
69 const StringMap &ProfileMap) {
70 if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
71 return EC;
72 if (std::error_code EC = writeFuncOffsetTable())
73 return EC;
6374 return sampleprof_error::success;
6475 }
6576
167178 return sampleprof_error::success;
168179 }
169180
181 std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
182 auto &OS = *OutputStream;
183
184 // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
185 auto &OFS = static_cast(OS);
186 uint64_t FuncOffsetTableStart = OS.tell();
187 if (OFS.seek(TableOffset) == (uint64_t)-1)
188 return sampleprof_error::ostream_seek_unsupported;
189 support::endian::Writer Writer(*OutputStream, support::little);
190 Writer.write(FuncOffsetTableStart);
191 if (OFS.seek(FuncOffsetTableStart) == (uint64_t)-1)
192 return sampleprof_error::ostream_seek_unsupported;
193
194 // Write out the table size.
195 encodeULEB128(FuncOffsetTable.size(), OS);
196
197 // Write out FuncOffsetTable.
198 for (auto entry : FuncOffsetTable) {
199 writeNameIdx(entry.first);
200 encodeULEB128(entry.second, OS);
201 }
202 return sampleprof_error::success;
203 }
204
170205 std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
171206 auto &OS = *OutputStream;
172207 std::set V;
211246 }
212247
213248 writeNameTable();
249 return sampleprof_error::success;
250 }
251
252 std::error_code SampleProfileWriterCompactBinary::writeHeader(
253 const StringMap &ProfileMap) {
254 support::endian::Writer Writer(*OutputStream, support::little);
255 if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
256 return EC;
257
258 // Reserve a slot for the offset of function offset table. The slot will
259 // be populated with the offset of FuncOffsetTable later.
260 TableOffset = OutputStream->tell();
261 Writer.write(static_cast(-2));
214262 return sampleprof_error::success;
215263 }
216264
282330 return writeBody(S);
283331 }
284332
333 std::error_code
334 SampleProfileWriterCompactBinary::write(const FunctionSamples &S) {
335 uint64_t Offset = OutputStream->tell();
336 StringRef Name = S.getName();
337 FuncOffsetTable[Name] = Offset;
338 encodeULEB128(S.getHeadSamples(), *OutputStream);
339 return writeBody(S);
340 }
341
285342 /// Create a sample profile file writer based on the specified format.
286343 ///
287344 /// \param Filename The file to create.
15141514 return false;
15151515 }
15161516 Reader = std::move(ReaderOrErr.get());
1517 Reader->collectFuncsToUse(M);
15171518 ProfileIsValid = (Reader->read() == sampleprof_error::success);
15181519 return true;
15191520 }
3535 namespace {
3636
3737 struct SampleProfTest : ::testing::Test {
38 std::string Data;
3938 LLVMContext Context;
39 std::string Profile;
4040 std::unique_ptr OS;
4141 std::unique_ptr Writer;
4242 std::unique_ptr Reader;
43 std::error_code EC;
4344
4445 SampleProfTest()
45 : Data(), OS(new raw_string_ostream(Data)), Writer(), Reader() {}
46 : Profile("profile"),
47 OS(new raw_fd_ostream(Profile, EC, sys::fs::F_None)), Writer(),
48 Reader() {}
4649
4750 void createWriter(SampleProfileFormat Format) {
4851 auto WriterOrErr = SampleProfileWriter::create(OS, Format);
5053 Writer = std::move(WriterOrErr.get());
5154 }
5255
53 void readProfile(std::unique_ptr &Profile) {
56 void readProfile(const Module &M) {
5457 auto ReaderOrErr = SampleProfileReader::create(Profile, Context);
5558 ASSERT_TRUE(NoError(ReaderOrErr.getError()));
5659 Reader = std::move(ReaderOrErr.get());
60 Reader->collectFuncsToUse(M);
5761 }
5862
5963 void testRoundTrip(SampleProfileFormat Format) {
8286 BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000);
8387 BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437);
8488
89 Module M("my_module", Context);
90 FunctionType *fn_type =
91 FunctionType::get(Type::getVoidTy(Context), {}, false);
92 M.getOrInsertFunction(FooName, fn_type);
93 M.getOrInsertFunction(BarName, fn_type);
94
8595 StringMap Profiles;
8696 Profiles[FooName] = std::move(FooSamples);
8797 Profiles[BarName] = std::move(BarSamples);
92102
93103 Writer->getOutputStream().flush();
94104
95 auto Profile = MemoryBuffer::getMemBufferCopy(Data);
96 readProfile(Profile);
105 readProfile(M);
97106
98107 EC = Reader->read();
99108 ASSERT_TRUE(NoError(EC));
163172 delete PS;
164173
165174 // Test that summary can be attached to and read back from module.
166 Module M("my_module", Context);
167175 M.setProfileSummary(MD);
168176 MD = M.getProfileSummary();
169177 ASSERT_TRUE(MD);