llvm.org GIT mirror llvm / 4f87d12
[PGO] Fix a bug in InstProfWriter addRecord For a new record with weight != 1, only edge profiling counters are scaled, VP data is not properly scaled. This patch refactors the code and fixes the problem. Also added sort by count interface (for follow up patch). git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@257143 91177308-0d34-0410-b5e6-96231b3b80d8 Xinliang David Li 3 years ago
4 changed file(s) with 135 addition(s) and 23 deletion(s). Raw diff Collapse all Expand all
354354 return left.Value < right.Value;
355355 });
356356 }
357 /// Sort ValueData Descending by Count
358 inline void sortByCount();
357359
358360 /// Merge data from another InstrProfValueSiteRecord
359361 /// Optionally scale merged counts by \p Weight.
360 instrprof_error mergeValueData(InstrProfValueSiteRecord &Input,
361 uint64_t Weight = 1);
362 instrprof_error merge(InstrProfValueSiteRecord &Input, uint64_t Weight = 1);
363 /// Scale up value profile data counts.
364 instrprof_error scale(uint64_t Weight);
362365 };
363366
364367 /// Profiling information for a single function.
401404 /// Optionally scale merged counts by \p Weight.
402405 instrprof_error merge(InstrProfRecord &Other, uint64_t Weight = 1);
403406
407 /// Scale up profile counts (including value profile data) by
408 /// \p Weight.
409 instrprof_error scale(uint64_t Weight);
410
411 /// Sort value profile data (per site) by count.
412 void sortValueData() {
413 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
414 std::vector &SiteRecords =
415 getValueSitesForKind(Kind);
416 for (auto &SR : SiteRecords)
417 SR.sortByCount();
418 }
419 }
404420 /// Clear value data entries
405421 void clearValueData() {
406422 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
435451 // Scale merged value counts by \p Weight.
436452 instrprof_error mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
437453 uint64_t Weight);
454 // Scale up value profile data count.
455 instrprof_error scaleValueProfData(uint32_t ValueKind, uint64_t Weight);
438456 };
439457
440458 uint32_t InstrProfRecord::getNumValueKinds() const {
502520 #define INSTR_PROF_VALUE_PROF_DATA
503521 #include "llvm/ProfileData/InstrProfData.inc"
504522
505 /*
506 * Initialize the record for runtime value profile data.
507 * Return 0 if the initialization is successful, otherwise
508 * return 1.
509 */
523 void InstrProfValueSiteRecord::sortByCount() {
524 ValueData.sort(
525 [](const InstrProfValueData &left, const InstrProfValueData &right) {
526 return left.Count > right.Count;
527 });
528 // Now truncate
529 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
530 if (ValueData.size() > max_s)
531 ValueData.resize(max_s);
532 }
533
534 /*
535 * Initialize the record for runtime value profile data.
536 * Return 0 if the initialization is successful, otherwise
537 * return 1.
538 */
510539 int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
511540 const uint16_t *NumValueSites,
512541 ValueProfNode **Nodes);
256256 return 0;
257257 }
258258
259 instrprof_error
260 InstrProfValueSiteRecord::mergeValueData(InstrProfValueSiteRecord &Input,
261 uint64_t Weight) {
259 instrprof_error InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
260 uint64_t Weight) {
262261 this->sortByTargetValues();
263262 Input.sortByTargetValues();
264263 auto I = ValueData.begin();
287286 return Result;
288287 }
289288
289 instrprof_error InstrProfValueSiteRecord::scale(uint64_t Weight) {
290 instrprof_error Result = instrprof_error::success;
291 for (auto I = ValueData.begin(), IE = ValueData.end(); I != IE; ++I) {
292 bool Overflowed;
293 I->Count = SaturatingMultiply(I->Count, Weight, &Overflowed);
294 if (Overflowed)
295 Result = instrprof_error::counter_overflow;
296 }
297 return Result;
298 }
299
290300 // Merge Value Profile data from Src record to this record for ValueKind.
291301 // Scale merged value counts by \p Weight.
292302 instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
302312 Src.getValueSitesForKind(ValueKind);
303313 instrprof_error Result = instrprof_error::success;
304314 for (uint32_t I = 0; I < ThisNumValueSites; I++)
305 MergeResult(Result,
306 ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I], Weight));
315 MergeResult(Result, ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight));
307316 return Result;
308317 }
309318
331340
332341 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
333342 MergeResult(Result, mergeValueProfData(Kind, Other, Weight));
343
344 return Result;
345 }
346
347 instrprof_error InstrProfRecord::scaleValueProfData(uint32_t ValueKind,
348 uint64_t Weight) {
349 uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
350 std::vector &ThisSiteRecords =
351 getValueSitesForKind(ValueKind);
352 instrprof_error Result = instrprof_error::success;
353 for (uint32_t I = 0; I < ThisNumValueSites; I++)
354 MergeResult(Result, ThisSiteRecords[I].scale(Weight));
355 return Result;
356 }
357
358 instrprof_error InstrProfRecord::scale(uint64_t Weight) {
359 instrprof_error Result = instrprof_error::success;
360 for (auto &Count : this->Counts) {
361 bool Overflowed;
362 Count = SaturatingMultiply(Count, Weight, &Overflowed);
363 if (Overflowed && Result == instrprof_error::success) {
364 Result = instrprof_error::counter_overflow;
365 }
366 }
367 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
368 MergeResult(Result, scaleValueProfData(Kind, Weight));
334369
335370 return Result;
336371 }
103103 ProfileDataMap.insert(std::make_pair(I.Hash, InstrProfRecord()));
104104 InstrProfRecord &Dest = Where->second;
105105
106 instrprof_error Result;
106 instrprof_error Result = instrprof_error::success;
107107 if (NewFunc) {
108108 // We've never seen a function with this name and hash, add it.
109109 Dest = std::move(I);
110110 // Fix up the name to avoid dangling reference.
111111 Dest.Name = FunctionData.find(Dest.Name)->getKey();
112 Result = instrprof_error::success;
113 if (Weight > 1) {
114 for (auto &Count : Dest.Counts) {
115 bool Overflowed;
116 Count = SaturatingMultiply(Count, Weight, &Overflowed);
117 if (Overflowed && Result == instrprof_error::success) {
118 Result = instrprof_error::counter_overflow;
119 }
120 }
121 }
112 if (Weight > 1)
113 Result = Dest.scale(Weight);
122114 } else {
123115 // We're updating a function we've seen before.
124116 Result = Dest.merge(I, Weight);
163163 [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
164164 return VD1.Count > VD2.Count;
165165 });
166
167 ASSERT_EQ(3U, VD[0].Count);
168 ASSERT_EQ(2U, VD[1].Count);
169 ASSERT_EQ(1U, VD[2].Count);
170
171 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
172 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
173 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
174 }
175
176 TEST_F(InstrProfTest, get_icall_data_read_write_with_weight) {
177 InstrProfRecord Record1("caller", 0x1234, {1, 2});
178 InstrProfRecord Record2("callee1", 0x1235, {3, 4});
179 InstrProfRecord Record3("callee2", 0x1235, {3, 4});
180 InstrProfRecord Record4("callee3", 0x1235, {3, 4});
181
182 // 4 value sites.
183 Record1.reserveSites(IPVK_IndirectCallTarget, 4);
184 InstrProfValueData VD0[] = {{(uint64_t) "callee1", 1},
185 {(uint64_t) "callee2", 2},
186 {(uint64_t) "callee3", 3}};
187 Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
188 // No value profile data at the second site.
189 Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
190 InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1},
191 {(uint64_t) "callee2", 2}};
192 Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
193 InstrProfValueData VD3[] = {{(uint64_t) "callee1", 1}};
194 Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
195
196 Writer.addRecord(std::move(Record1), 10);
197 Writer.addRecord(std::move(Record2));
198 Writer.addRecord(std::move(Record3));
199 Writer.addRecord(std::move(Record4));
200 auto Profile = Writer.writeBuffer();
201 readProfile(std::move(Profile));
202
203 ErrorOr R = Reader->getInstrProfRecord("caller", 0x1234);
204 ASSERT_TRUE(NoError(R.getError()));
205 ASSERT_EQ(4U, R.get().getNumValueSites(IPVK_IndirectCallTarget));
206 ASSERT_EQ(3U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
207 ASSERT_EQ(0U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
208 ASSERT_EQ(2U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
209 ASSERT_EQ(1U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
210
211 std::unique_ptr VD =
212 R.get().getValueForSite(IPVK_IndirectCallTarget, 0);
213 // Now sort the target acording to frequency.
214 std::sort(&VD[0], &VD[3],
215 [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
216 return VD1.Count > VD2.Count;
217 });
218 ASSERT_EQ(30U, VD[0].Count);
219 ASSERT_EQ(20U, VD[1].Count);
220 ASSERT_EQ(10U, VD[2].Count);
221
166222 ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
167223 ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
168224 ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));