llvm.org GIT mirror llvm / 450ef2a
Prototype: Reduce llvm-profdata merge memory usage further The InstrProfWriter already stores the name and hash of the record in the nested maps it uses for lookup while merging - this data is duplicated in the value within the maps. Refactor the InstrProfRecord to use a nested struct for the counters themselves so that InstrProfWriter can use this nested struct alone without the name or hash duplicated there. This work is incomplete, but enough to demonstrate the value (around a 50% decrease in memory usage for a large test case (10GB -> 5GB)). Though most of that decrease is probably from removing the SoftInstrProfError as well, but I haven't implemented a replacement for it yet. (it needs to go with the counters, because the operations on the counters - merging, etc, are where the failures are - unlike the name/hash which are totally unused by those counter-related operations and thus easy to split out) Ongoing discussion about removing SoftInstrProfError as a field of the InstrProfRecord is happening on the thread that added it - including the possibility of moving back towards an earlier version of that proposed patch that passed SoftInstrProfError through the various APIs, rather than as a member of InstrProfRecord. Reviewers: davidxl Differential Revision: https://reviews.llvm.org/D34838 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@307298 91177308-0d34-0410-b5e6-96231b3b80d8 David Blaikie 2 years ago
8 changed file(s) with 83 addition(s) and 72 deletion(s). Raw diff Collapse all Expand all
248248
249249 /// Same as the above interface but using an ArrayRef, as well as \p Sum.
250250 void annotateValueSite(Module &M, Instruction &Inst,
251 ArrayRef VDs,
252 uint64_t Sum, InstrProfValueKind ValueKind,
253 uint32_t MaxMDCount);
251 ArrayRef VDs, uint64_t Sum,
252 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
254253
255254 /// Extract the value profile data from \p Inst which is annotated with
256255 /// value profile meta data. Return false if there is no value data annotated,
589588
590589 /// Profiling information for a single function.
591590 struct InstrProfRecord {
592 StringRef Name;
593 uint64_t Hash;
594591 std::vector Counts;
595592 SoftInstrProfErrors SIPE;
596593
597594 InstrProfRecord() = default;
598 InstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts)
599 : Name(Name), Hash(Hash), Counts(std::move(Counts)) {}
595 InstrProfRecord(std::vector Counts) : Counts(std::move(Counts)) {}
600596 InstrProfRecord(InstrProfRecord &&) = default;
601597 InstrProfRecord(const InstrProfRecord &RHS)
602 : Name(RHS.Name), Hash(RHS.Hash), Counts(RHS.Counts), SIPE(RHS.SIPE),
598 : Counts(RHS.Counts),
603599 ValueData(RHS.ValueData
604600 ? llvm::make_unique(*RHS.ValueData)
605601 : nullptr) {}
606602 InstrProfRecord &operator=(InstrProfRecord &&) = default;
607603 InstrProfRecord &operator=(const InstrProfRecord &RHS) {
608 Name = RHS.Name;
609 Hash = RHS.Hash;
610604 Counts = RHS.Counts;
611 SIPE = RHS.SIPE;
612605 if (!RHS.ValueData) {
613606 ValueData = nullptr;
614607 return *this;
625618 /// Return the number of value profile kinds with non-zero number
626619 /// of profile sites.
627620 inline uint32_t getNumValueKinds() const;
628
629621 /// Return the number of instrumented sites for ValueKind.
630622 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
631623
741733
742734 // Scale up value profile data count.
743735 void scaleValueProfData(uint32_t ValueKind, uint64_t Weight);
736 };
737
738 struct NamedInstrProfRecord : InstrProfRecord {
739 StringRef Name;
740 uint64_t Hash;
741
742 NamedInstrProfRecord() = default;
743 NamedInstrProfRecord(StringRef Name, uint64_t Hash,
744 std::vector Counts)
745 : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
744746 };
745747
746748 uint32_t InstrProfRecord::getNumValueKinds() const {
3939
4040 /// A file format agnostic iterator over profiling data.
4141 class InstrProfIterator : public std::iterator
42 InstrProfRecord> {
42 NamedInstrProfRecord> {
4343 InstrProfReader *Reader = nullptr;
44 InstrProfRecord Record;
44 value_type Record;
4545
4646 void Increment();
4747
5252 InstrProfIterator &operator++() { Increment(); return *this; }
5353 bool operator==(const InstrProfIterator &RHS) { return Reader == RHS.Reader; }
5454 bool operator!=(const InstrProfIterator &RHS) { return Reader != RHS.Reader; }
55 InstrProfRecord &operator*() { return Record; }
56 InstrProfRecord *operator->() { return &Record; }
55 value_type &operator*() { return Record; }
56 value_type *operator->() { return &Record; }
5757 };
5858
5959 /// Base class and interface for reading profiling data of any known instrprof
60 /// format. Provides an iterator over InstrProfRecords.
60 /// format. Provides an iterator over NamedInstrProfRecords.
6161 class InstrProfReader {
6262 instrprof_error LastError = instrprof_error::success;
6363
6969 virtual Error readHeader() = 0;
7070
7171 /// Read a single record.
72 virtual Error readNextRecord(InstrProfRecord &Record) = 0;
72 virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0;
7373
7474 /// Iterator over profile data.
7575 InstrProfIterator begin() { return InstrProfIterator(this); }
160160 Error readHeader() override;
161161
162162 /// Read a single record.
163 Error readNextRecord(InstrProfRecord &Record) override;
163 Error readNextRecord(NamedInstrProfRecord &Record) override;
164164
165165 InstrProfSymtab &getSymtab() override {
166166 assert(Symtab.get());
208208
209209 static bool hasFormat(const MemoryBuffer &DataBuffer);
210210 Error readHeader() override;
211 Error readNextRecord(InstrProfRecord &Record) override;
211 Error readNextRecord(NamedInstrProfRecord &Record) override;
212212
213213 bool isIRLevelProfile() const override {
214214 return (Version & VARIANT_MASK_IR_PROF) != 0;
242242 return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t));
243243 }
244244
245 Error readName(InstrProfRecord &Record);
246 Error readFuncHash(InstrProfRecord &Record);
245 Error readName(NamedInstrProfRecord &Record);
246 Error readFuncHash(NamedInstrProfRecord &Record);
247247 Error readRawCounts(InstrProfRecord &Record);
248248 Error readValueProfilingData(InstrProfRecord &Record);
249249 bool atEnd() const { return Data == DataEnd; }
280280 /// Trait for lookups into the on-disk hash table for the binary instrprof
281281 /// format.
282282 class InstrProfLookupTrait {
283 std::vector<InstrProfRecord> DataBuffer;
283 std::vector<NamedInstrProfRecord> DataBuffer;
284284 IndexedInstrProf::HashT HashType;
285285 unsigned FormatVersion;
286286 // Endianness of the input value profile data.
292292 InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
293293 : HashType(HashType), FormatVersion(FormatVersion) {}
294294
295 using data_type = ArrayRef<InstrProfRecord>;
295 using data_type = ArrayRef<NamedInstrProfRecord>;
296296
297297 using internal_key_type = StringRef;
298298 using external_key_type = StringRef;
333333
334334 // Read all the profile records with the same key pointed to the current
335335 // iterator.
336 virtual Error getRecords(ArrayRef<InstrProfRecord> &Data) = 0;
336 virtual Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) = 0;
337337
338338 // Read all the profile records with the key equal to FuncName
339339 virtual Error getRecords(StringRef FuncName,
340 ArrayRef<InstrProfRecord> &Data) = 0;
340 ArrayRef<NamedInstrProfRecord> &Data) = 0;
341341 virtual void advanceToNextKey() = 0;
342342 virtual bool atEnd() const = 0;
343343 virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
363363 IndexedInstrProf::HashT HashType, uint64_t Version);
364364 ~InstrProfReaderIndex() override = default;
365365
366 Error getRecords(ArrayRef<InstrProfRecord> &Data) override;
366 Error getRecords(ArrayRef<NamedInstrProfRecord> &Data) override;
367367 Error getRecords(StringRef FuncName,
368 ArrayRef<InstrProfRecord> &Data) override;
368 ArrayRef<NamedInstrProfRecord> &Data) override;
369369 void advanceToNextKey() override { RecordIterator++; }
370370
371371 bool atEnd() const override {
418418 /// Read the file header.
419419 Error readHeader() override;
420420 /// Read a single record.
421 Error readNextRecord(InstrProfRecord &Record) override;
422
423 /// Return the pointer to InstrProfRecord associated with FuncName
424 /// and FuncHash
421 Error readNextRecord(NamedInstrProfRecord &Record) override;
422
423 /// Return the NamedInstrProfRecord associated with FuncName and FuncHash
425424 Expected getInstrProfRecord(StringRef FuncName,
426425 uint64_t FuncHash);
427426
3232
3333 class InstrProfWriter {
3434 public:
35 using ProfilingData = SmallDenseMap;
35 using ProfilingData =
36 SmallDenseMap;
3637 enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel };
3738
3839 private:
4950 /// Add function counts for the given function. If there are already counts
5051 /// for this function and the hash and number of counts match, each counter is
5152 /// summed. Optionally scale counts by \p Weight.
52 Error addRecord(InstrProfRecord &&I, uint64_t Weight = 1);
53 Error addRecord(NamedInstrProfRecord &&I, uint64_t Weight = 1);
5354
5455 /// Merge existing function counts from the given writer.
5556 Error mergeRecordsFromWriter(InstrProfWriter &&IPW);
6162 Error writeText(raw_fd_ostream &OS);
6263
6364 /// Write \c Record in text format to \c OS
64 static void writeRecordInText(const InstrProfRecord &Record,
65 static void writeRecordInText(StringRef Name, uint64_t Hash,
66 const InstrProfRecord &Counters,
6567 InstrProfSymtab &Symtab, raw_fd_ostream &OS);
6668
6769 /// Write the profile, returning the raw data. For testing.
8486 void setOutputSparse(bool Sparse);
8587
8688 private:
89 Error addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I,
90 uint64_t Weight = 1);
8791 bool shouldEncodeData(const ProfilingData &PD);
8892 void writeImpl(ProfOStream &OS);
8993 };
1616
1717 #include "llvm/ADT/ArrayRef.h"
1818 #include "llvm/IR/ProfileSummary.h"
19 #include "llvm/ProfileData/InstrProf.h"
1920 #include "llvm/Support/Error.h"
2021 #include
2122 #include
2526 #include
2627
2728 namespace llvm {
28
29 struct InstrProfRecord;
3029
3130 namespace sampleprof {
3231
220220 #undef VP_READ_ADVANCE
221221 }
222222
223 Error TextInstrProfReader::readNextRecord(InstrProfRecord &Record) {
223 Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
224224 // Skip empty lines and comments.
225225 while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
226226 ++Line;
376376 }
377377
378378 template
379 Error RawInstrProfReader::readName(InstrProfRecord &Record) {
379 Error RawInstrProfReader::readName(NamedInstrProfRecord &Record) {
380380 Record.Name = getName(Data->NameRef);
381381 return success();
382382 }
383383
384384 template
385 Error RawInstrProfReader::readFuncHash(InstrProfRecord &Record) {
385 Error RawInstrProfReader::readFuncHash(NamedInstrProfRecord &Record) {
386386 Record.Hash = swap(Data->FuncHash);
387387 return success();
388388 }
444444 }
445445
446446 template
447 Error RawInstrProfReader::readNextRecord(InstrProfRecord &Record) {
447 Error RawInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
448448 if (atEnd())
449449 // At this point, ValueDataStart field points to the next header.
450450 if (Error E = readNextHeader(getNextHeaderPos()))
549549
550550 template
551551 Error InstrProfReaderIndex::getRecords(
552 StringRef FuncName, ArrayRef<InstrProfRecord> &Data) {
552 StringRef FuncName, ArrayRef<NamedInstrProfRecord> &Data) {
553553 auto Iter = HashTable->find(FuncName);
554554 if (Iter == HashTable->end())
555555 return make_error(instrprof_error::unknown_function);
563563
564564 template
565565 Error InstrProfReaderIndex::getRecords(
566 ArrayRef<InstrProfRecord> &Data) {
566 ArrayRef<NamedInstrProfRecord> &Data) {
567567 if (atEnd())
568568 return make_error(instrprof_error::eof);
569569
643643
644644 InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
645645 // FIXME: This only computes an empty summary. Need to call addRecord for
646 // all InstrProfRecords to get the correct summary.
646 // all NamedInstrProfRecords to get the correct summary.
647647 this->Summary = Builder.getSummary();
648648 return Cur;
649649 }
706706 Expected
707707 IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName,
708708 uint64_t FuncHash) {
709 ArrayRef<InstrProfRecord> Data;
709 ArrayRef<NamedInstrProfRecord> Data;
710710 Error Err = Index->getRecords(FuncName, Data);
711711 if (Err)
712712 return std::move(Err);
731731 return success();
732732 }
733733
734 Error IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) {
734 Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
735735 static unsigned RecordIndex = 0;
736736
737 ArrayRef<InstrProfRecord> Data;
737 ArrayRef<NamedInstrProfRecord> Data;
738738
739739 Error E = Index->getRecords(Data);
740740 if (E)
175175 this->Sparse = Sparse;
176176 }
177177
178 Error InstrProfWriter::addRecord(InstrProfRecord &&I, uint64_t Weight) {
179 auto &ProfileDataMap = FunctionData[I.Name];
178 Error InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight) {
179 auto Name = I.Name;
180 auto Hash = I.Hash;
181 return addRecord(Name, Hash, std::move(I), Weight);
182 }
183
184 Error InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,
185 InstrProfRecord &&I, uint64_t Weight) {
186 auto &ProfileDataMap = FunctionData[Name];
180187
181188 bool NewFunc;
182189 ProfilingData::iterator Where;
183190 std::tie(Where, NewFunc) =
184 ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord()));
191 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
185192 InstrProfRecord &Dest = Where->second;
186193
187194 if (NewFunc) {
188195 // We've never seen a function with this name and hash, add it.
189196 Dest = std::move(I);
190 // Fix up the name to avoid dangling reference.
191 Dest.Name = FunctionData.find(Dest.Name)->getKey();
192197 if (Weight > 1)
193198 Dest.scale(Weight);
194199 } else {
204209 Error InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW) {
205210 for (auto &I : IPW.FunctionData)
206211 for (auto &Func : I.getValue())
207 if (Error E = addRecord(std::move(Func.second), 1))
212 if (Error E = addRecord(I.getKey(), Func.first, std::move(Func.second)))
208213 return E;
209214 return Error::success();
210215 }
322327 #include "llvm/ProfileData/InstrProfData.inc"
323328 };
324329
325 void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func,
330 void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
331 const InstrProfRecord &Func,
326332 InstrProfSymtab &Symtab,
327333 raw_fd_ostream &OS) {
328 OS << Func.Name << "\n";
329 OS << "# Func Hash:\n" << Func.Hash << "\n";
334 OS << Name << "\n";
335 OS << "# Func Hash:\n" << Hash << "\n";
330336 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
331337 OS << "# Counter Values:\n";
332338 for (uint64_t Count : Func.Counts)
374380 for (const auto &I : FunctionData)
375381 if (shouldEncodeData(I.getValue()))
376382 for (const auto &Func : I.getValue())
377 writeRecordInText(Func.second, Symtab, OS);
383 writeRecordInText(I.getKey(), Func.first, Func.second, Symtab, OS);
378384 return Error::success();
379385 }
527527
528528 if (doTextFormatDump) {
529529 InstrProfSymtab &Symtab = Reader->getSymtab();
530 InstrProfWriter::writeRecordInText(Func, Symtab, OS);
530 InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab,
531 OS);
531532 continue;
532533 }
533534
224224 static const char callee6[] = "callee6";
225225
226226 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write) {
227 InstrProfRecord Record1("caller", 0x1234, {1, 2});
227 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
228228
229229 // 4 value sites.
230230 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
268268 }
269269
270270 TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
271 InstrProfRecord Record("caller", 0x1234, {1, 2});
271 NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
272272 Record.reserveSites(IPVK_IndirectCallTarget, 1);
273273 InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
274274 {4000, 4}, {6000, 6}};
364364 }
365365
366366 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_with_weight) {
367 InstrProfRecord Record1("caller", 0x1234, {1, 2});
367 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
368368
369369 // 4 value sites.
370370 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
407407 }
408408
409409 TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
410 InstrProfRecord Record1("caller", 0x1234, {1, 2});
410 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
411411
412412 // 4 value sites.
413413 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
455455
456456 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
457457 static const char caller[] = "caller";
458 InstrProfRecord Record11(caller, 0x1234, {1, 2});
459 InstrProfRecord Record12(caller, 0x1234, {1, 2});
458 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
459 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
460460
461461 // 5 value sites.
462462 Record11.reserveSites(IPVK_IndirectCallTarget, 5);
576576 ASSERT_EQ(InstrProfError::take(std::move(Result3)),
577577 instrprof_error::success);
578578
579 InstrProfRecord Record4("baz", 0x5678, {3, 4});
579 NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
580580 Record4.reserveSites(IPVK_IndirectCallTarget, 1);
581581 InstrProfValueData VD4[] = {{uint64_t(bar), 1}};
582582 Record4.addValueData(IPVK_IndirectCallTarget, 0, VD4, 1, nullptr);
585585 instrprof_error::success);
586586
587587 // Verify value data counter overflow.
588 InstrProfRecord Record5("baz", 0x5678, {5, 6});
588 NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
589589 Record5.reserveSites(IPVK_IndirectCallTarget, 1);
590590 InstrProfValueData VD5[] = {{uint64_t(bar), Max}};
591591 Record5.addValueData(IPVK_IndirectCallTarget, 0, VD5, 1, nullptr);
618618 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge_site_trunc) {
619619 static const char caller[] = "caller";
620620
621 InstrProfRecord Record11(caller, 0x1234, {1, 2});
622 InstrProfRecord Record12(caller, 0x1234, {1, 2});
621 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
622 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
623623
624624 // 2 value sites.
625625 Record11.reserveSites(IPVK_IndirectCallTarget, 2);
685685 }
686686
687687 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write) {
688 InstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
688 InstrProfRecord SrcRecord({1ULL << 31, 2});
689689 addValueProfData(SrcRecord);
690690 std::unique_ptr VPData =
691691 ValueProfData::serializeFrom(SrcRecord);
692692
693 InstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
693 InstrProfRecord Record({1ULL << 31, 2});
694694 VPData->deserializeTo(Record, nullptr);
695695
696696 // Now read data from Record and sanity check the data
751751
752752 TEST_P(MaybeSparseInstrProfTest, value_prof_data_read_write_mapping) {
753753
754 InstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
754 NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
755755 addValueProfData(SrcRecord);
756756 std::unique_ptr VPData =
757757 ValueProfData::serializeFrom(SrcRecord);
758758
759 InstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
759 NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
760760 InstrProfSymtab Symtab;
761761 Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);
762762 Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);