llvm.org GIT mirror llvm / b725085
[llvm-profdata] Add support for weighted merge of profile data This change adds support for an optional weight when merging profile data with the llvm-profdata tool. Weights are specified by adding an option ':<weight>' suffix to the input file names. Adding support for arbitrary weighting of input profile data allows for relative importance to be placed on the input data from multiple training runs. Both sampled and instrumented profiles are supported. Reviewers: dnovillo, bogner, davidxl Subscribers: llvm-commits Differential Revision: http://reviews.llvm.org/D14547 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@254669 91177308-0d34-0410-b5e6-96231b3b80d8 Nathan Slingerland 3 years ago
13 changed file(s) with 266 addition(s) and 47 deletion(s). Raw diff Collapse all Expand all
2727 SYNOPSIS
2828 ^^^^^^^^
2929
30 :program:`llvm-profdata merge` [*options*] [*filenames...*]
30 :program:`llvm-profdata merge` [*options*] [*filename[:weight]...*]
3131
3232 DESCRIPTION
3333 ^^^^^^^^^^^
3535 :program:`llvm-profdata merge` takes several profile data files
3636 generated by PGO instrumentation and merges them together into a single
3737 indexed profile data file.
38
39 The profile counts in each input file can be scaled (multiplied) by specifying
40 ``:``, where `` is a decimal integer >= 1.
41 A default weight of 1 is assumed if only `` is given.
3842
3943 OPTIONS
4044 ^^^^^^^
217217 }
218218
219219 /// Merge data from another InstrProfValueSiteRecord
220 void mergeValueData(InstrProfValueSiteRecord &Input) {
220 /// Optionally scale merged counts by \p Weight.
221 void mergeValueData(InstrProfValueSiteRecord &Input, uint64_t Weight = 1) {
221222 this->sortByTargetValues();
222223 Input.sortByTargetValues();
223224 auto I = ValueData.begin();
227228 while (I != IE && I->Value < J->Value)
228229 ++I;
229230 if (I != IE && I->Value == J->Value) {
230 I->Count = SaturatingAdd(I->Count, J->Count);
231 // TODO: Check for counter overflow and return error if it occurs.
232 uint64_t JCount = J->Count;
233 if (Weight > 1)
234 JCount = SaturatingMultiply(JCount, Weight);
235 I->Count = SaturatingAdd(I->Count, JCount);
231236 ++I;
232237 continue;
233238 }
273278 ValueMapType *HashKeys);
274279
275280 /// Merge the counts in \p Other into this one.
276 inline instrprof_error merge(InstrProfRecord &Other);
281 /// Optionally scale merged counts by \p Weight.
282 inline instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1);
277283
278284 /// Used by InstrProfWriter: update the value strings to commoned strings in
279285 /// the writer instance.
325331 }
326332
327333 // Merge Value Profile data from Src record to this record for ValueKind.
328 instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src) {
334 // Scale merged value counts by \p Weight.
335 instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
336 uint64_t Weight) {
329337 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
330338 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
331339 if (ThisNumValueSites != OtherNumValueSites)
335343 std::vector &OtherSiteRecords =
336344 Src.getValueSitesForKind(ValueKind);
337345 for (uint32_t I = 0; I < ThisNumValueSites; I++)
338 ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I]);
346 ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I], Weight);
339347 return instrprof_error::success;
340348 }
341349 };
421429 VData.Value = (uint64_t)StrTab->insertString((const char *)VData.Value);
422430 }
423431
424 instrprof_error InstrProfRecord::merge(InstrProfRecord &Other) {
432 instrprof_error InstrProfRecord::merge(InstrProfRecord &Other,
433 uint64_t Weight) {
425434 // If the number of counters doesn't match we either have bad data
426435 // or a hash collision.
427436 if (Counts.size() != Other.Counts.size())
431440
432441 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
433442 bool ResultOverflowed;
434 Counts[I] = SaturatingAdd(Counts[I], Other.Counts[I], ResultOverflowed);
443 uint64_t OtherCount = Other.Counts[I];
444 if (Weight > 1) {
445 OtherCount = SaturatingMultiply(OtherCount, Weight, ResultOverflowed);
446 if (ResultOverflowed)
447 Result = instrprof_error::counter_overflow;
448 }
449 Counts[I] = SaturatingAdd(Counts[I], OtherCount, ResultOverflowed);
435450 if (ResultOverflowed)
436451 Result = instrprof_error::counter_overflow;
437452 }
438453
439454 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
440 instrprof_error MergeValueResult = mergeValueProfData(Kind, Other);
455 instrprof_error MergeValueResult = mergeValueProfData(Kind, Other, Weight);
441456 if (MergeValueResult != instrprof_error::success)
442457 Result = MergeValueResult;
443458 }
3838 void updateStringTableReferences(InstrProfRecord &I);
3939 /// Add function counts for the given function. If there are already counts
4040 /// for this function and the hash and number of counts match, each counter is
41 /// summed.
42 std::error_code addRecord(InstrProfRecord &&I);
41 /// summed. Optionally scale counts by \p Weight.
42 std::error_code addRecord(InstrProfRecord &&I, uint64_t Weight = 1);
4343 /// Write the profile to \c OS
4444 void write(raw_fd_ostream &OS);
4545 /// Write the profile in text format to \c OS
172172 SampleRecord() : NumSamples(0), CallTargets() {}
173173
174174 /// Increment the number of samples for this record by \p S.
175 /// Optionally scale sample count \p S by \p Weight.
175176 ///
176177 /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
177178 /// around unsigned integers.
178 void addSamples(uint64_t S) {
179 void addSamples(uint64_t S, uint64_t Weight = 1) {
180 if (Weight > 1)
181 S = SaturatingMultiply(S, Weight);
179182 NumSamples = SaturatingAdd(NumSamples, S);
180183 }
181184
182185 /// Add called function \p F with samples \p S.
186 /// Optionally scale sample count \p S by \p Weight.
183187 ///
184188 /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
185189 /// around unsigned integers.
186 void addCalledTarget(StringRef F, uint64_t S) {
190 void addCalledTarget(StringRef F, uint64_t S, uint64_t Weight = 1) {
187191 uint64_t &TargetSamples = CallTargets[F];
192 if (Weight > 1)
193 S = SaturatingMultiply(S, Weight);
188194 TargetSamples = SaturatingAdd(TargetSamples, S);
189195 }
190196
195201 const CallTargetMap &getCallTargets() const { return CallTargets; }
196202
197203 /// Merge the samples in \p Other into this record.
198 void merge(const SampleRecord &Other) {
199 addSamples(Other.getSamples());
204 /// Optionally scale sample counts by \p Weight.
205 void merge(const SampleRecord &Other, uint64_t Weight = 1) {
206 addSamples(Other.getSamples(), Weight);
200207 for (const auto &I : Other.getCallTargets())
201 addCalledTarget(I.first(), I.second);
208 addCalledTarget(I.first(), I.second, Weight);
202209 }
203210
204211 void print(raw_ostream &OS, unsigned Indent) const;
225232 FunctionSamples() : TotalSamples(0), TotalHeadSamples(0) {}
226233 void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;
227234 void dump() const;
228 void addTotalSamples(uint64_t Num) { TotalSamples += Num; }
229 void addHeadSamples(uint64_t Num) { TotalHeadSamples += Num; }
230 void addBodySamples(uint32_t LineOffset, uint32_t Discriminator,
231 uint64_t Num) {
232 BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(Num);
235 void addTotalSamples(uint64_t Num, uint64_t Weight = 1) {
236 if (Weight > 1)
237 Num = SaturatingMultiply(Num, Weight);
238 TotalSamples += Num;
239 }
240 void addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
241 if (Weight > 1)
242 Num = SaturatingMultiply(Num, Weight);
243 TotalHeadSamples += Num;
244 }
245 void addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num,
246 uint64_t Weight = 1) {
247 BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(Num,
248 Weight);
233249 }
234250 void addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator,
235 std::string FName, uint64_t Num) {
236 BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(FName,
237 Num);
251 std::string FName, uint64_t Num,
252 uint64_t Weight = 1) {
253 BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
254 FName, Num, Weight);
238255 }
239256
240257 /// Return the number of samples collected at the given location.
283300 }
284301
285302 /// Merge the samples in \p Other into this one.
286 void merge(const FunctionSamples &Other) {
287 addTotalSamples(Other.getTotalSamples());
288 addHeadSamples(Other.getHeadSamples());
303 /// Optionally scale samples by \p Weight.
304 void merge(const FunctionSamples &Other, uint64_t Weight = 1) {
305 addTotalSamples(Other.getTotalSamples(), Weight);
306 addHeadSamples(Other.getHeadSamples(), Weight);
289307 for (const auto &I : Other.getBodySamples()) {
290308 const LineLocation &Loc = I.first;
291309 const SampleRecord &Rec = I.second;
292 BodySamples[Loc].merge(Rec);
310 BodySamples[Loc].merge(Rec, Weight);
293311 }
294312 for (const auto &I : Other.getCallsiteSamples()) {
295313 const CallsiteLocation &Loc = I.first;
296314 const FunctionSamples &Rec = I.second;
297 functionSamplesAt(Loc).merge(Rec);
315 functionSamplesAt(Loc).merge(Rec, Weight);
298316 }
299317 }
300318
9797 I.updateStrings(&StringTable);
9898 }
9999
100 std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I) {
100 std::error_code InstrProfWriter::addRecord(InstrProfRecord &&I,
101 uint64_t Weight) {
101102 updateStringTableReferences(I);
102103 auto &ProfileDataMap = FunctionData[I.Name];
103104
112113 // We've never seen a function with this name and hash, add it.
113114 Dest = std::move(I);
114115 Result = instrprof_error::success;
116 if (Weight > 1) {
117 for (auto &Count : Dest.Counts) {
118 bool Overflowed;
119 Count = SaturatingMultiply(Count, Weight, Overflowed);
120 if (Overflowed && Result == instrprof_error::success) {
121 Result = instrprof_error::counter_overflow;
122 }
123 }
124 }
115125 } else {
116126 // We're updating a function we've seen before.
117 Result = Dest.merge(I);
127 Result = Dest.merge(I, Weight);
118128 }
119129
120130 // We keep track of the max function count as we go for simplicity.
0 bar:1772037:35370
1 17: 35370
2 18: 35370
3 19: 7005
4 20: 29407
5 21: 12170
6 23: 18150 bar:19829
7 25: 36666
0 foo:1763288:35327
1 7: 35327
2 8: 35327
3 9: 6930
4 10: 29341
5 11: 11906
6 13: 18185 foo:19531
7 15: 36458
0 Tests for weighted merge of instrumented profiles.
1
2 1- Merge the foo and bar profiles with unity weight and verify the combined output
3 RUN: llvm-profdata merge --instr %p/Inputs/weight-instr-bar.profdata:1 %p/Inputs/weight-instr-foo.profdata:1 -o %t
4 RUN: llvm-profdata show --instr -all-functions %t | FileCheck %s --check-prefix=WEIGHT1
5 WEIGHT1: Counters:
6 WEIGHT1: usage:
7 WEIGHT1: Hash: 0x0000000000000000
8 WEIGHT1: Counters: 1
9 WEIGHT1: Function count: 0
10 WEIGHT1: foo:
11 WEIGHT1: Hash: 0x000000000000028a
12 WEIGHT1: Counters: 3
13 WEIGHT1: Function count: 866988873
14 WEIGHT1: bar:
15 WEIGHT1: Hash: 0x000000000000028a
16 WEIGHT1: Counters: 3
17 WEIGHT1: Function count: 866988873
18 WEIGHT1: main:
19 WEIGHT1: Hash: 0x7d31c47ea98f8248
20 WEIGHT1: Counters: 60
21 WEIGHT1: Function count: 2
22 WEIGHT1: Functions shown: 4
23 WEIGHT1: Total functions: 4
24 WEIGHT1: Maximum function count: 866988873
25 WEIGHT1: Maximum internal block count: 267914296
26
27 2- Merge the foo and bar profiles with weight 3x and 5x respectively and verify the combined output
28 RUN: llvm-profdata merge --instr %p/Inputs/weight-instr-bar.profdata:3 %p/Inputs/weight-instr-foo.profdata:5 -o %t
29 RUN: llvm-profdata show --instr -all-functions %t | FileCheck %s --check-prefix=WEIGHT2
30 WEIGHT2: Counters:
31 WEIGHT2: usage:
32 WEIGHT2: Hash: 0x0000000000000000
33 WEIGHT2: Counters: 1
34 WEIGHT2: Function count: 0
35 WEIGHT2: foo:
36 WEIGHT2: Hash: 0x000000000000028a
37 WEIGHT2: Counters: 3
38 WEIGHT2: Function count: 4334944365
39 WEIGHT2: bar:
40 WEIGHT2: Hash: 0x000000000000028a
41 WEIGHT2: Counters: 3
42 WEIGHT2: Function count: 2600966619
43 WEIGHT2: main:
44 WEIGHT2: Hash: 0x7d31c47ea98f8248
45 WEIGHT2: Counters: 60
46 WEIGHT2: Function count: 8
47 WEIGHT2: Functions shown: 4
48 WEIGHT2: Total functions: 4
49 WEIGHT2: Maximum function count: 4334944365
50 WEIGHT2: Maximum internal block count: 1339571480
51
52 3- Bad merge: foo and bar profiles with invalid weights
53 RUN: not llvm-profdata merge --instr %p/Inputs/weight-instr-bar.profdata:3 %p/Inputs/weight-instr-foo.profdata:-5 -o %t.out 2>&1 | FileCheck %s --check-prefix=ERROR3
54 ERROR3: error: Input weight must be a positive integer.
0 Tests for weighted merge of sample profiles.
1
2 1- Merge the foo and bar profiles with unity weight and verify the combined output
3 RUN: llvm-profdata merge --sample --text %p/Inputs/weight-sample-bar.proftext:1 %p/Inputs/weight-sample-foo.proftext:1 -o - | FileCheck %s --check-prefix=WEIGHT1
4 WEIGHT1: foo:1763288:35327
5 WEIGHT1: 7: 35327
6 WEIGHT1: 8: 35327
7 WEIGHT1: 9: 6930
8 WEIGHT1: 10: 29341
9 WEIGHT1: 11: 11906
10 WEIGHT1: 13: 18185 foo:19531
11 WEIGHT1: 15: 36458
12 WEIGHT1: bar:1772037:35370
13 WEIGHT1: 17: 35370
14 WEIGHT1: 18: 35370
15 WEIGHT1: 19: 7005
16 WEIGHT1: 20: 29407
17 WEIGHT1: 21: 12170
18 WEIGHT1: 23: 18150 bar:19829
19 WEIGHT1: 25: 36666
20
21 2- Merge the foo and bar profiles with weight 3x and 5x respectively and verify the combined output
22 RUN: llvm-profdata merge --sample --text %p/Inputs/weight-sample-bar.proftext:3 %p/Inputs/weight-sample-foo.proftext:5 -o - | FileCheck %s --check-prefix=WEIGHT2
23 WEIGHT2: foo:8816440:176635
24 WEIGHT2: 7: 176635
25 WEIGHT2: 8: 176635
26 WEIGHT2: 9: 34650
27 WEIGHT2: 10: 146705
28 WEIGHT2: 11: 59530
29 WEIGHT2: 13: 90925 foo:97655
30 WEIGHT2: 15: 182290
31 WEIGHT2: bar:5316111:106110
32 WEIGHT2: 17: 106110
33 WEIGHT2: 18: 106110
34 WEIGHT2: 19: 21015
35 WEIGHT2: 20: 88221
36 WEIGHT2: 21: 36510
37 WEIGHT2: 23: 54450 bar:59487
38 WEIGHT2: 25: 109998
39
40 3- Bad merge: foo and bar profiles with invalid weights
41 RUN: not llvm-profdata merge --sample --text %p/Inputs/weight-sample-bar.proftext:3 %p/Inputs/weight-sample-foo.proftext:-5 -o %t.out 2>&1 | FileCheck %s --check-prefix=ERROR3
42 ERROR3: error: Input weight must be a positive integer.
1111 //===----------------------------------------------------------------------===//
1212
1313 #include "llvm/ADT/SmallSet.h"
14 #include "llvm/ADT/SmallVector.h"
1415 #include "llvm/ADT/StringRef.h"
1516 #include "llvm/IR/LLVMContext.h"
1617 #include "llvm/ProfileData/InstrProfReader.h"
2627 #include "llvm/Support/PrettyStackTrace.h"
2728 #include "llvm/Support/Signals.h"
2829 #include "llvm/Support/raw_ostream.h"
30 #include
2931
3032 using namespace llvm;
3133
9294 }
9395 }
9496
95 static void mergeInstrProfile(const cl::list &Inputs,
97 struct WeightedFile {
98 StringRef Filename;
99 uint64_t Weight;
100
101 WeightedFile() {}
102
103 WeightedFile(StringRef F, uint64_t W) : Filename{F}, Weight{W} {}
104 };
105 typedef SmallVector WeightedFileVector;
106
107 static void mergeInstrProfile(const WeightedFileVector &Inputs,
96108 StringRef OutputFilename,
97109 ProfileFormat OutputFormat) {
98110 if (OutputFilename.compare("-") == 0)
108120
109121 InstrProfWriter Writer;
110122 SmallSet WriterErrorCodes;
111 for (const auto &Filename : Inputs) {
112 auto ReaderOrErr = InstrProfReader::create(Filename);
123 for (const auto &Input : Inputs) {
124 auto ReaderOrErr = InstrProfReader::create(Input.Filename);
113125 if (std::error_code ec = ReaderOrErr.getError())
114 exitWithErrorCode(ec, Filename);
126 exitWithErrorCode(ec, Input.Filename);
115127
116128 auto Reader = std::move(ReaderOrErr.get());
117129 for (auto &I : *Reader) {
118 if (std::error_code EC = Writer.addRecord(std::move(I))) {
130 if (std::error_code EC = Writer.addRecord(std::move(I), Input.Weight)) {
119131 // Only show hint the first time an error occurs.
120132 bool firstTime = WriterErrorCodes.insert(EC).second;
121 handleMergeWriterError(EC, Filename, I.Name, firstTime);
133 handleMergeWriterError(EC, Input.Filename, I.Name, firstTime);
122134 }
123135 }
124136 if (Reader->hasError())
125 exitWithErrorCode(Reader->getError(), Filename);
137 exitWithErrorCode(Reader->getError(), Input.Filename);
126138 }
127139 if (OutputFormat == PF_Text)
128140 Writer.writeText(Output);
134146 sampleprof::SPF_None, sampleprof::SPF_Text, sampleprof::SPF_Binary,
135147 sampleprof::SPF_GCC};
136148
137 static void mergeSampleProfile(const cl::list &Inputs,
149 static void mergeSampleProfile(const WeightedFileVector &Inputs,
138150 StringRef OutputFilename,
139151 ProfileFormat OutputFormat) {
140152 using namespace sampleprof;
146158 auto Writer = std::move(WriterOrErr.get());
147159 StringMap ProfileMap;
148160 SmallVector, 5> Readers;
149 for (const auto &Filename : Inputs) {
161 for (const auto &Input : Inputs) {
150162 auto ReaderOrErr =
151 SampleProfileReader::create(Filename, getGlobalContext());
163 SampleProfileReader::create(Input.Filename, getGlobalContext());
152164 if (std::error_code EC = ReaderOrErr.getError())
153 exitWithErrorCode(EC, Filename);
165 exitWithErrorCode(EC, Input.Filename);
154166
155167 // We need to keep the readers around until after all the files are
156168 // read so that we do not lose the function names stored in each
159171 Readers.push_back(std::move(ReaderOrErr.get()));
160172 const auto Reader = Readers.back().get();
161173 if (std::error_code EC = Reader->read())
162 exitWithErrorCode(EC, Filename);
174 exitWithErrorCode(EC, Input.Filename);
163175
164176 StringMap &Profiles = Reader->getProfiles();
165177 for (StringMap::iterator I = Profiles.begin(),
167179 I != E; ++I) {
168180 StringRef FName = I->first();
169181 FunctionSamples &Samples = I->second;
170 ProfileMap[FName].merge(Samples);
182 ProfileMap[FName].merge(Samples, Input.Weight);
171183 }
172184 }
173185 Writer->write(ProfileMap);
186 }
187
188 static void parseInputFiles(const cl::list &Inputs,
189 WeightedFileVector &WeightedInputs) {
190 WeightedInputs.reserve(Inputs.size());
191
192 for (StringRef Input : Inputs) {
193 StringRef FileName;
194 StringRef WeightStr;
195 std::tie(FileName, WeightStr) = Input.rsplit(':');
196 if (WeightStr.empty() || sys::fs::exists(Input)) {
197 // No weight specified or valid path containing delimiter.
198 WeightedInputs.push_back(WeightedFile(Input, 1));
199 } else {
200 // Input weight specified.
201 uint64_t Weight;
202 if (WeightStr.getAsInteger(10, Weight) || Weight < 1) {
203 // Invalid input weight.
204 exitWithError("Input weight must be a positive integer.");
205 }
206 WeightedInputs.push_back(WeightedFile(FileName, Weight));
207 }
208 }
174209 }
175210
176211 static int merge_main(int argc, const char *argv[]) {
177212 cl::list Inputs(cl::Positional, cl::Required, cl::OneOrMore,
178 cl::desc("s...>"));
213 cl::desc("[:weight]...>"));
179214
180215 cl::opt OutputFilename("output", cl::value_desc("output"),
181216 cl::init("-"), cl::Required,
197232
198233 cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n");
199234
235 WeightedFileVector WeightedInputs;
236 parseInputFiles(Inputs, WeightedInputs);
237
200238 if (ProfileKind == instr)
201 mergeInstrProfile(Inputs, OutputFilename, OutputFormat);
239 mergeInstrProfile(WeightedInputs, OutputFilename, OutputFormat);
202240 else
203 mergeSampleProfile(Inputs, OutputFilename, OutputFormat);
241 mergeSampleProfile(WeightedInputs, OutputFilename, OutputFormat);
204242
205243 return 0;
206244 }
489489 ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount());
490490 }
491491
492 TEST_F(InstrProfTest, get_weighted_function_counts) {
493 InstrProfRecord Record1("foo", 0x1234, {1, 2});
494 InstrProfRecord Record2("foo", 0x1235, {3, 4});
495 Writer.addRecord(std::move(Record1), 3);
496 Writer.addRecord(std::move(Record2), 5);
497 auto Profile = Writer.writeBuffer();
498 readProfile(std::move(Profile));
499
500 std::vector Counts;
501 ASSERT_TRUE(NoError(Reader->getFunctionCounts("foo", 0x1234, Counts)));
502 ASSERT_EQ(2U, Counts.size());
503 ASSERT_EQ(3U, Counts[0]);
504 ASSERT_EQ(6U, Counts[1]);
505
506 ASSERT_TRUE(NoError(Reader->getFunctionCounts("foo", 0x1235, Counts)));
507 ASSERT_EQ(2U, Counts.size());
508 ASSERT_EQ(15U, Counts[0]);
509 ASSERT_EQ(20U, Counts[1]);
510 }
511
492512 } // end anonymous namespace