llvm.org GIT mirror llvm / 3b1e430
Support for remapping profile data when symbols change, for instrumentation-based profiling. Reviewers: davidxl, tejohnson, dlj, erik.pilkington Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D51247 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@344184 91177308-0d34-0410-b5e6-96231b3b80d8 Richard Smith 1 year, 10 days ago
3 changed file(s) with 223 addition(s) and 15 deletion(s). Raw diff Collapse all Expand all
348348 OnDiskIterableChainedHashTable;
349349
350350 template
351 class InstrProfReaderItaniumRemapper;
352
353 template
351354 class InstrProfReaderIndex : public InstrProfReaderIndexBase {
352355 private:
353356 std::unique_ptr HashTable;
354357 typename HashTableImpl::data_iterator RecordIterator;
355358 uint64_t FormatVersion;
356359
360 friend class InstrProfReaderItaniumRemapper;
361
357362 public:
358363 InstrProfReaderIndex(const unsigned char *Buckets,
359364 const unsigned char *const Payload,
385390 }
386391 };
387392
393 /// Name matcher supporting fuzzy matching of symbol names to names in profiles.
394 class InstrProfReaderRemapper {
395 public:
396 virtual ~InstrProfReaderRemapper() {}
397 virtual Error populateRemappings() { return Error::success(); }
398 virtual Error getRecords(StringRef FuncName,
399 ArrayRef &Data) = 0;
400 };
401
388402 /// Reader for the indexed binary instrprof format.
389403 class IndexedInstrProfReader : public InstrProfReader {
390404 private:
391405 /// The profile data file contents.
392406 std::unique_ptr DataBuffer;
407 /// The profile remapping file contents.
408 std::unique_ptr RemappingBuffer;
393409 /// The index into the profile data.
394410 std::unique_ptr Index;
411 /// The profile remapping file contents.
412 std::unique_ptr Remapper;
395413 /// Profile summary data.
396414 std::unique_ptr Summary;
397415 // Index to the current record in the record array.
403421 const unsigned char *Cur);
404422
405423 public:
406 IndexedInstrProfReader(std::unique_ptr DataBuffer)
407 : DataBuffer(std::move(DataBuffer)), RecordIndex(0) {}
424 IndexedInstrProfReader(
425 std::unique_ptr DataBuffer,
426 std::unique_ptr RemappingBuffer = nullptr)
427 : DataBuffer(std::move(DataBuffer)),
428 RemappingBuffer(std::move(RemappingBuffer)), RecordIndex(0) {}
408429 IndexedInstrProfReader(const IndexedInstrProfReader &) = delete;
409430 IndexedInstrProfReader &operator=(const IndexedInstrProfReader &) = delete;
410431
433454
434455 /// Factory method to create an indexed reader.
435456 static Expected>
436 create(const Twine &Path);
457 create(const Twine &Path, const Twine &RemappingPath = "");
437458
438459 static Expected>
439 create(std::unique_ptr Buffer);
460 create(std::unique_ptr Buffer,
461 std::unique_ptr RemappingBuffer = nullptr);
440462
441463 // Used for testing purpose only.
442464 void setValueProfDataEndianness(support::endianness Endianness) {
1313
1414 #include "llvm/ProfileData/InstrProfReader.h"
1515 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
1617 #include "llvm/ADT/STLExtras.h"
1718 #include "llvm/ADT/StringRef.h"
1819 #include "llvm/IR/ProfileSummary.h"
2223 #include "llvm/Support/Error.h"
2324 #include "llvm/Support/ErrorOr.h"
2425 #include "llvm/Support/MemoryBuffer.h"
26 #include "llvm/Support/SymbolRemappingReader.h"
2527 #include "llvm/Support/SwapByteOrder.h"
2628 #include
2729 #include
8789 }
8890
8991 Expected>
90 IndexedInstrProfReader::create(const Twine &Path) {
92 IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) {
9193 // Set up the buffer to read.
9294 auto BufferOrError = setupMemoryBuffer(Path);
9395 if (Error E = BufferOrError.takeError())
9496 return std::move(E);
95 return IndexedInstrProfReader::create(std::move(BufferOrError.get()));
97
98 // Set up the remapping buffer if requested.
99 std::unique_ptr RemappingBuffer;
100 std::string RemappingPathStr = RemappingPath.str();
101 if (!RemappingPathStr.empty()) {
102 auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr);
103 if (Error E = RemappingBufferOrError.takeError())
104 return std::move(E);
105 RemappingBuffer = std::move(RemappingBufferOrError.get());
106 }
107
108 return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
109 std::move(RemappingBuffer));
96110 }
97111
98112 Expected>
99 IndexedInstrProfReader::create(std::unique_ptr Buffer) {
113 IndexedInstrProfReader::create(std::unique_ptr Buffer,
114 std::unique_ptr RemappingBuffer) {
100115 // Sanity check the buffer.
101116 if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits::max())
102117 return make_error(instrprof_error::too_large);
104119 // Create the reader.
105120 if (!IndexedInstrProfReader::hasFormat(*Buffer))
106121 return make_error(instrprof_error::bad_magic);
107 auto Result = llvm::make_unique(std::move(Buffer));
122 auto Result = llvm::make_unique(
123 std::move(Buffer), std::move(RemappingBuffer));
108124
109125 // Initialize the reader and return the result.
110126 if (Error E = initializeReader(*Result))
586602 RecordIterator = HashTable->data_begin();
587603 }
588604
605 namespace {
606 /// A remapper that does not apply any remappings.
607 class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
608 InstrProfReaderIndexBase &Underlying;
609
610 public:
611 InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
612 : Underlying(Underlying) {}
613
614 Error getRecords(StringRef FuncName,
615 ArrayRef &Data) override {
616 return Underlying.getRecords(FuncName, Data);
617 }
618 };
619 }
620
621 /// A remapper that applies remappings based on a symbol remapping file.
622 template
623 class llvm::InstrProfReaderItaniumRemapper
624 : public InstrProfReaderRemapper {
625 public:
626 InstrProfReaderItaniumRemapper(
627 std::unique_ptr RemapBuffer,
628 InstrProfReaderIndex &Underlying)
629 : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
630 }
631
632 /// Extract the original function name from a PGO function name.
633 static StringRef extractName(StringRef Name) {
634 // We can have multiple :-separated pieces; there can be pieces both
635 // before and after the mangled name. Find the first part that starts
636 // with '_Z'; we'll assume that's the mangled name we want.
637 std::pair Parts = {StringRef(), Name};
638 while (true) {
639 Parts = Parts.second.split(':');
640 if (Parts.first.startswith("_Z"))
641 return Parts.first;
642 if (Parts.second.empty())
643 return Name;
644 }
645 }
646
647 /// Given a mangled name extracted from a PGO function name, and a new
648 /// form for that mangled name, reconstitute the name.
649 static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
650 StringRef Replacement,
651 SmallVectorImpl &Out) {
652 Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
653 Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
654 Out.insert(Out.end(), Replacement.begin(), Replacement.end());
655 Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
656 }
657
658 Error populateRemappings() override {
659 if (Error E = Remappings.read(*RemapBuffer))
660 return E;
661 for (StringRef Name : Underlying.HashTable->keys()) {
662 StringRef RealName = extractName(Name);
663 if (auto Key = Remappings.insert(RealName)) {
664 // FIXME: We could theoretically map the same equivalence class to
665 // multiple names in the profile data. If that happens, we should
666 // return NamedInstrProfRecords from all of them.
667 MappedNames.insert({Key, RealName});
668 }
669 }
670 return Error::success();
671 }
672
673 Error getRecords(StringRef FuncName,
674 ArrayRef &Data) override {
675 StringRef RealName = extractName(FuncName);
676 if (auto Key = Remappings.lookup(RealName)) {
677 StringRef Remapped = MappedNames.lookup(Key);
678 if (!Remapped.empty()) {
679 if (RealName.begin() == FuncName.begin() &&
680 RealName.end() == FuncName.end())
681 FuncName = Remapped;
682 else {
683 // Try rebuilding the name from the given remapping.
684 SmallString<256> Reconstituted;
685 reconstituteName(FuncName, RealName, Remapped, Reconstituted);
686 Error E = Underlying.getRecords(Reconstituted, Data);
687 if (!E)
688 return E;
689
690 // If we failed because the name doesn't exist, fall back to asking
691 // about the original name.
692 if (Error Unhandled = handleErrors(
693 std::move(E), [](std::unique_ptr Err) {
694 return Err->get() == instrprof_error::unknown_function
695 ? Error::success()
696 : Error(std::move(Err));
697 }))
698 return Unhandled;
699 }
700 }
701 }
702 return Underlying.getRecords(FuncName, Data);
703 }
704
705 private:
706 /// The memory buffer containing the remapping configuration. Remappings
707 /// holds pointers into this buffer.
708 std::unique_ptr RemapBuffer;
709
710 /// The mangling remapper.
711 SymbolRemappingReader Remappings;
712
713 /// Mapping from mangled name keys to the name used for the key in the
714 /// profile data.
715 /// FIXME: Can we store a location within the on-disk hash table instead of
716 /// redoing lookup?
717 DenseMap MappedNames;
718
719 /// The real profile data reader.
720 InstrProfReaderIndex &Underlying;
721 };
722
589723 bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
590724 using namespace support;
591725
682816 uint64_t HashOffset = endian::byte_swap(Header->HashOffset);
683817
684818 // The rest of the file is an on disk hash table.
685 InstrProfReaderIndexBase *IndexPtr = nullptr;
686 IndexPtr = new InstrProfReaderIndex(
687 Start + HashOffset, Cur, Start, HashType, FormatVersion);
688 Index.reset(IndexPtr);
819 auto IndexPtr =
820 llvm::make_unique>(
821 Start + HashOffset, Cur, Start, HashType, FormatVersion);
822
823 // Load the remapping table now if requested.
824 if (RemappingBuffer) {
825 Remapper = llvm::make_unique<
826 InstrProfReaderItaniumRemapper>(
827 std::move(RemappingBuffer), *IndexPtr);
828 if (Error E = Remapper->populateRemappings())
829 return E;
830 } else {
831 Remapper = llvm::make_unique(*IndexPtr);
832 }
833 Index = std::move(IndexPtr);
834
689835 return success();
690836 }
691837
706852 IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName,
707853 uint64_t FuncHash) {
708854 ArrayRef Data;
709 Error Err = Index->getRecords(FuncName, Data);
855 Error Err = Remapper->getRecords(FuncName, Data);
710856 if (Err)
711857 return std::move(Err);
712858 // Found it. Look for counters with the right hash.
4141
4242 void SetUp() { Writer.setOutputSparse(false); }
4343
44 void readProfile(std::unique_ptr Profile) {
45 auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile));
44 void readProfile(std::unique_ptr Profile,
45 std::unique_ptr Remapping = nullptr) {
46 auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),
47 std::move(Remapping));
4648 EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
4749 Reader = std::move(ReaderOrErr.get());
4850 }
989991 }
990992 }
991993
994 TEST_P(MaybeSparseInstrProfTest, remapping_test) {
995 Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);
996 Writer.addRecord({"file:_Z3barf", 0x567, {5, 6, 7}}, Err);
997 auto Profile = Writer.writeBuffer();
998 readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(
999 type i l
1000 name 3bar 4quux
1001 )"));
1002
1003 std::vector Counts;
1004 for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1005 EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1006 Succeeded());
1007 ASSERT_EQ(4u, Counts.size());
1008 EXPECT_EQ(1u, Counts[0]);
1009 EXPECT_EQ(2u, Counts[1]);
1010 EXPECT_EQ(3u, Counts[2]);
1011 EXPECT_EQ(4u, Counts[3]);
1012 }
1013
1014 for (StringRef BarName : {"file:_Z3barf", "file:_Z4quuxf"}) {
1015 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1016 Succeeded());
1017 ASSERT_EQ(3u, Counts.size());
1018 EXPECT_EQ(5u, Counts[0]);
1019 EXPECT_EQ(6u, Counts[1]);
1020 EXPECT_EQ(7u, Counts[2]);
1021 }
1022
1023 for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1024 "_Z3barf", "otherfile:_Z4quuxf"}) {
1025 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1026 Failed());
1027 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1028 Failed());
1029 }
1030 }
1031
9921032 TEST_F(SparseInstrProfTest, preserve_no_records) {
9931033 Writer.addRecord({"foo", 0x1234, {0}}, Err);
9941034 Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);