llvm.org GIT mirror llvm / 9312179
[PGO] Implement ValueProfiling Closure interfaces for runtime value profile data This is one of the many steps to commonize value profiling support between profile runtime and compiler/llvm tools. After this change, profiler runtime now can share the same C APIs to do VP serialization/deseriazation with LLVM host tools (and produces value data in identical format between indexed and raw profile). It is not yet enabled in profiler runtime yet. Also added a unit test case to test runtime profile data serialization/deserialization interfaces implemented using common closure code. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@254110 91177308-0d34-0410-b5e6-96231b3b80d8 Xinliang David Li 3 years ago
3 changed file(s) with 270 addition(s) and 10 deletion(s). Raw diff Collapse all Expand all
552552 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
553553 } ValueProfRecordClosure;
554554
555 /// Return the \c ValueProfRecord header size including the padding bytes.
555 /* A wrapper struct that represents value profile runtime data.
556 * Like InstrProfRecord class which is used by profiling host tools,
557 * ValueProfRuntimeRecord also implements the abstract intefaces defined in
558 * ValueProfRecordClosure so that the runtime data can be serialized using
559 * shared C implementation. In this structure, NumValueSites and Nodes
560 * members are the primary fields while other fields hold the derived
561 * information for fast implementation of closure interfaces.
562 */
563 typedef struct ValueProfRuntimeRecord {
564 /* Number of sites for each value profile kind. */
565 uint16_t *NumValueSites;
566 /* An array of linked-list headers. The size of of the array is the
567 * total number of value profile sites : sum(NumValueSites[*])). Each
568 * linked-list stores the values profiled for a value profile site. */
569 ValueProfNode **Nodes;
570
571 /* Total number of value profile kinds which have at least one
572 * value profile sites. */
573 uint32_t NumValueKinds;
574 /* An array recording the number of values tracked at each site.
575 * The size of the array is TotalNumValueSites.
576 */
577 uint8_t *SiteCountArray[IPVK_Last + 1];
578 ValueProfNode **NodesKind[IPVK_Last + 1];
579 } ValueProfRuntimeRecord;
580
581 /* Initialize the record for runtime value profile data. */
582 void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
583 uint16_t *NumValueSites,
584 ValueProfNode **Nodes);
585
586 /* Release memory allocated for the runtime record. */
587 void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord);
588
589 /* Return the size of ValueProfData structure that can be used to store
590 the value profile data collected at runtime. */
591 uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record);
592
593 /* Return a ValueProfData instance that stores the data collected at runtime. */
594 ValueProfData *
595 serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record);
596
597
598 /*! \brief Return the \c ValueProfRecord header size including the
599 * padding bytes.
600 */
556601 inline uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
557602 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
558603 sizeof(uint8_t) * NumValueSites;
561606 return Size;
562607 }
563608
564 /// Return the total size of the value profile record including the
565 /// header and the value data.
609 /*! \brief Return the total size of the value profile record including the
610 * header and the value data.
611 */
566612 inline uint32_t getValueProfRecordSize(uint32_t NumValueSites,
567613 uint32_t NumValueData) {
568614 return getValueProfRecordHeaderSize(NumValueSites) +
569615 sizeof(InstrProfValueData) * NumValueData;
570616 }
571617
572 /// Return the pointer to the start of value data array.
618 /*! \brief Return the pointer to the start of value data array.
619 */
573620 inline InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
574621 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
575622 This->NumValueSites));
576623 }
577624
578 /// Return the total number of value data for \c This record.
625 /*! \brief Return the total number of value data for \c This record.
626 */
579627 inline uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
580628 uint32_t NumValueData = 0;
581629 uint32_t I;
584632 return NumValueData;
585633 }
586634
587 /// Use this method to advance to the next \c This \c ValueProfRecord.
635 /* \brief Use this method to advance to the next \c This \c ValueProfRecord.
636 */
588637 inline ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
589638 uint32_t NumValueData = getValueProfRecordNumValueData(This);
590639 return (ValueProfRecord *)((char *)This +
592641 NumValueData));
593642 }
594643
595 /// Return the first \c ValueProfRecord instance.
644 /*! \brief Return the first \c ValueProfRecord instance.
645 */
596646 inline ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
597647 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
598648 }
649
650
599651
600652 namespace IndexedInstrProf {
601653
187187 return VPD;
188188 }
189189
190 // C wrappers of InstrProfRecord member functions used in Closure.
191 // These C wrappers are used as adaptors so that C++ code can be
192 // invoked as callbacks.
190 /*! \brief ValueProfRecordClosure Interface implementation for InstrProfRecord
191 * class. These C wrappers are used as adaptors so that C++ code can be
192 * invoked as callbacks.
193 */
193194 uint32_t getNumValueKindsInstrProf(const void *Record) {
194195 return reinterpret_cast(Record)->getNumValueKinds();
195196 }
261262 return VPD;
262263 }
263264
265 /* The value profiler runtime library stores the value profile data
266 * for a given function in NumValueSites and Nodes. This is the
267 * method to initialize the RuntimeRecord with the runtime data to
268 * pre-compute the information needed to efficiently implement
269 * ValueProfRecordClosure's callback interfaces.
270 */
271 void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
272 uint16_t *NumValueSites,
273 ValueProfNode **Nodes) {
274 unsigned I, J, S = 0, NumValueKinds = 0;
275 RuntimeRecord->NumValueSites = NumValueSites;
276 RuntimeRecord->Nodes = Nodes;
277 for (I = 0; I <= IPVK_Last; I++) {
278 uint16_t N = NumValueSites[I];
279 if (!N) {
280 RuntimeRecord->SiteCountArray[I] = 0;
281 continue;
282 }
283 NumValueKinds++;
284 RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1);
285 RuntimeRecord->NodesKind[I] = &RuntimeRecord->Nodes[S];
286 for (J = 0; J < N; J++) {
287 uint8_t C = 0;
288 ValueProfNode *Site = RuntimeRecord->Nodes[S + J];
289 while (Site) {
290 C++;
291 Site = Site->Next;
292 }
293 if (C > UCHAR_MAX)
294 C = UCHAR_MAX;
295 RuntimeRecord->SiteCountArray[I][J] = C;
296 }
297 S += N;
298 }
299 RuntimeRecord->NumValueKinds = NumValueKinds;
300 }
301
302 void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) {
303 unsigned I;
304 for (I = 0; I <= IPVK_Last; I++) {
305 if (RuntimeRecord->SiteCountArray[I])
306 free(RuntimeRecord->SiteCountArray[I]);
307 }
308 }
309
310 /* ValueProfRecordClosure Interface implementation for
311 * ValueProfDataRuntimeRecord. */
312 uint32_t getNumValueKindsRT(const void *R) {
313 return ((const ValueProfRuntimeRecord *)R)->NumValueKinds;
314 }
315
316 uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
317 return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK];
318 }
319
320 uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) {
321 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
322 return Record->SiteCountArray[VK][S];
323 }
324
325 uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
326 unsigned I, S = 0;
327 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
328 if (Record->SiteCountArray[VK] == 0)
329 return 0;
330 for (I = 0; I < Record->NumValueSites[VK]; I++)
331 S += Record->SiteCountArray[VK][I];
332 return S;
333 }
334
335 void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK,
336 uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) {
337 unsigned I, N = 0;
338 const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
339 N = getNumValueDataForSiteRT(R, VK, S);
340 ValueProfNode *VNode = Record->NodesKind[VK][S];
341 for (I = 0; I < N; I++) {
342 Dst[I] = VNode->VData;
343 VNode = VNode->Next;
344 }
345 }
346
347 ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) {
348 return (ValueProfData *)calloc(TotalSizeInBytes, 1);
349 }
350
351 static ValueProfRecordClosure RTRecordClosure = {0,
352 getNumValueKindsRT,
353 getNumValueSitesRT,
354 getNumValueDataRT,
355 getNumValueDataForSiteRT,
356 0,
357 getValueForSiteRT,
358 allocValueProfDataRT};
359
360 /* Return the size of ValueProfData structure to store data
361 * recorded in the runtime record.
362 */
363 uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
364 RTRecordClosure.Record = Record;
365 return getValueProfDataSize(&RTRecordClosure);
366 }
367
368 /* Return a ValueProfData instance that stores the data collected
369 from runtime. */
370 ValueProfData *
371 serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record) {
372 RTRecordClosure.Record = Record;
373 return serializeValueProfDataFrom(&RTRecordClosure);
374 }
375
376
377
378
264379 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
265380 InstrProfRecord::ValueMapType *VMap) {
266381 Record.reserveSites(Kind, NumValueSites);
272387 ValueData += ValueDataCount;
273388 }
274389 }
390
275391 // For writing/serializing, Old is the host endianness, and New is
276392 // byte order intended on disk. For Reading/deserialization, Old
277393 // is the on-disk source endianness, and New is the host endianness.
381381 ASSERT_EQ(Max, VD[0].Count);
382382 }
383383
384 // Synthesize runtime value profile data.
385 ValueProfNode Site1Values[5] = {{{uint64_t("callee1"), 400}, &Site1Values[1]},
386 {{uint64_t("callee2"), 1000}, &Site1Values[2]},
387 {{uint64_t("callee3"), 500}, &Site1Values[3]},
388 {{uint64_t("callee4"), 300}, &Site1Values[4]},
389 {{uint64_t("callee5"), 100}, 0}};
390
391 ValueProfNode Site2Values[4] = {{{uint64_t("callee5"), 800}, &Site2Values[1]},
392 {{uint64_t("callee3"), 1000}, &Site2Values[2]},
393 {{uint64_t("callee2"), 2500}, &Site2Values[3]},
394 {{uint64_t("callee1"), 1300}, 0}};
395
396 ValueProfNode Site3Values[3] = {{{uint64_t("callee6"), 800}, &Site3Values[1]},
397 {{uint64_t("callee3"), 1000}, &Site3Values[2]},
398 {{uint64_t("callee4"), 5500}, 0}};
399
400 ValueProfNode Site4Values[2] = {{{uint64_t("callee2"), 1800}, &Site4Values[1]},
401 {{uint64_t("callee3"), 2000}, 0}};
402
403 static ValueProfNode *ValueProfNodes[5] = {&Site1Values[0], &Site2Values[0],
404 &Site3Values[0], &Site4Values[0], 0};
405 static uint16_t NumValueSites[IPVK_Last + 1] = {5};
406 TEST_F(InstrProfTest, runtime_value_prof_data_read_write) {
407 ValueProfRuntimeRecord RTRecord;
408 initializeValueProfRuntimeRecord(&RTRecord, &NumValueSites[0],
409 &ValueProfNodes[0]);
410
411 ValueProfData *VPData = serializeValueProfDataFromRT(&RTRecord);
412
413 InstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
414
415 VPData->deserializeTo(Record, 0);
416
417 // Now read data from Record and sanity check the data
418 ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
419 ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
420 ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
421 ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
422 ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
423 ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
424
425 auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
426 return VD1.Count > VD2.Count;
427 };
428 std::unique_ptr VD_0(
429 Record.getValueForSite(IPVK_IndirectCallTarget, 0));
430 std::sort(&VD_0[0], &VD_0[5], Cmp);
431 ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
432 ASSERT_EQ(1000U, VD_0[0].Count);
433 ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
434 ASSERT_EQ(500U, VD_0[1].Count);
435 ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
436 ASSERT_EQ(400U, VD_0[2].Count);
437 ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
438 ASSERT_EQ(300U, VD_0[3].Count);
439 ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
440 ASSERT_EQ(100U, VD_0[4].Count);
441
442 std::unique_ptr VD_1(
443 Record.getValueForSite(IPVK_IndirectCallTarget, 1));
444 std::sort(&VD_1[0], &VD_1[4], Cmp);
445 ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
446 ASSERT_EQ(2500U, VD_1[0].Count);
447 ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
448 ASSERT_EQ(1300U, VD_1[1].Count);
449 ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
450 ASSERT_EQ(1000U, VD_1[2].Count);
451 ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
452 ASSERT_EQ(800U, VD_1[3].Count);
453
454 std::unique_ptr VD_2(
455 Record.getValueForSite(IPVK_IndirectCallTarget, 2));
456 std::sort(&VD_2[0], &VD_2[3], Cmp);
457 ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
458 ASSERT_EQ(5500U, VD_2[0].Count);
459 ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
460 ASSERT_EQ(1000U, VD_2[1].Count);
461 ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
462 ASSERT_EQ(800U, VD_2[2].Count);
463
464 std::unique_ptr VD_3(
465 Record.getValueForSite(IPVK_IndirectCallTarget, 3));
466 std::sort(&VD_3[0], &VD_3[2], Cmp);
467 ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
468 ASSERT_EQ(2000U, VD_3[0].Count);
469 ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
470 ASSERT_EQ(1800U, VD_3[1].Count);
471
472 finalizeValueProfRuntimeRecord(&RTRecord);
473 free(VPData);
474 }
475
384476 TEST_F(InstrProfTest, get_max_function_count) {
385477 InstrProfRecord Record1("foo", 0x1234, {1ULL << 31, 2});
386478 InstrProfRecord Record2("bar", 0, {1ULL << 63});