llvm.org GIT mirror llvm / ad650ee
[profile] Static counter allocation for value profiling (part-1) Differential Revision: http://reviews.llvm.org/D20459 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@270336 91177308-0d34-0410-b5e6-96231b3b80d8 Xinliang David Li 4 years ago
5 changed file(s) with 163 addition(s) and 15 deletion(s). Raw diff Collapse all Expand all
5757 : INSTR_PROF_DATA_SECT_NAME_STR;
5858 }
5959
60 /// Return the name of data section containing pointers to value profile
61 /// counters/nodes.
62 inline StringRef getInstrProfValuesSectionName(bool AddSegment) {
63 return AddSegment ? "__DATA," INSTR_PROF_VALS_SECT_NAME_STR
64 : INSTR_PROF_VALS_SECT_NAME_STR;
65 }
66
67 /// Return the name of data section containing nodes holdling value
68 /// profiling data.
69 inline StringRef getInstrProfVNodesSectionName(bool AddSegment) {
70 return AddSegment ? "__DATA," INSTR_PROF_VNODES_SECT_NAME_STR
71 : INSTR_PROF_VNODES_SECT_NAME_STR;
72 }
73
6074 /// Return the name profile runtime entry point to do value profiling
6175 /// for a given site.
6276 inline StringRef getInstrProfValueProfFuncName() {
7892
7993 /// Return the name prefix of profile counter variables.
8094 inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
95
96 /// Return the name prefix of value profile variables.
97 inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
98
99 /// Return the name of value profile node array variables:
100 inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
81101
82102 /// Return the name prefix of the COMDAT group for instrumentation variables
83103 /// associated with a COMDAT function.
7979 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
8080 FunctionAddr)
8181 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
82 ConstantPointerNull::get(Int8PtrTy))
82 ValuesPtrExpr)
8383 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
8484 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
8585 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
596596 #define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data
597597 #define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names
598598 #define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts
599 /* Array of pointers. Each pointer points to a list
600 * of value nodes associated with one value site.
601 */
602 #define INSTR_PROF_VALS_SECT_NAME __llvm_prf_values
603 /* Value profile nodes section. */
604 #define INSTR_PROF_VNODES_SECT_NAME __llvm_prf_vnodes
599605 #define INSTR_PROF_COVMAP_SECT_NAME __llvm_covmap
600606
601607 #define INSTR_PROF_DATA_SECT_NAME_STR \
606612 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
607613 #define INSTR_PROF_COVMAP_SECT_NAME_STR \
608614 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME)
615 #define INSTR_PROF_VALS_SECT_NAME_STR \
616 INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME)
617 #define INSTR_PROF_VNODES_SECT_NAME_STR \
618 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME)
609619
610620 /* Macros to define start/stop section symbol for a given
611621 * section on Linux. For instance
8383 /// Emit the section with compressed function names.
8484 void emitNameData();
8585
86 /// Emit value nodes section for value profiling.
87 void emitVNodes();
88
8689 /// Emit runtime registration functions for each profile data variable.
8790 void emitRegistration();
8891
3030 cl::desc("Enable name string compression"),
3131 cl::init(true));
3232
33 cl::opt ValueProfileStaticAlloc(
34 "vp-static-alloc",
35 cl::desc("Do static counter allocation for value profiler"),
36 cl::init(true));
37 cl::opt NumCountersPerValueSite(
38 "vp-counters-per-site",
39 cl::desc("The average number of profile counters allocated "
40 "per value profiling site."),
41 // This is set to a very small value because in real programs, only
42 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
43 // For those sites with non-zero profile, the average number of targets
44 // is usually smaller than 2.
45 cl::init(1.0));
46
3347 class InstrProfilingLegacyPass : public ModulePass {
3448 InstrProfiling InstrProf;
3549
140154 if (!MadeChange)
141155 return false;
142156
157 emitVNodes();
143158 emitNameData();
144159 emitRegistration();
145160 emitRuntimeHook();
287302 return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
288303 }
289304
305 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
306 // Don't do this for Darwin. compiler-rt uses linker magic.
307 if (Triple(M.getTargetTriple()).isOSDarwin())
308 return false;
309
310 // Use linker script magic to get data/cnts/name start/end.
311 if (Triple(M.getTargetTriple()).isOSLinux() ||
312 Triple(M.getTargetTriple()).isOSFreeBSD() ||
313 Triple(M.getTargetTriple()).isPS4CPU())
314 return false;
315
316 return true;
317 }
318
290319 GlobalVariable *
291320 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
292321 GlobalVariable *NamePtr = Inc->getName();
320349 CounterPtr->setAlignment(8);
321350 CounterPtr->setComdat(ProfileVarsComdat);
322351
352 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
353 // Allocate statically the array of pointers to value profile nodes for
354 // the current function.
355 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
356 if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
357
358 uint64_t NS = 0;
359 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
360 NS += PD.NumValueSites[Kind];
361 if (NS) {
362 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
363
364 auto *ValuesVar =
365 new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
366 Constant::getNullValue(ValuesTy),
367 getVarName(Inc, getInstrProfValuesVarPrefix()));
368 ValuesVar->setVisibility(NamePtr->getVisibility());
369 ValuesVar->setSection(getInstrProfValuesSectionName(isMachO()));
370 ValuesVar->setAlignment(8);
371 ValuesVar->setComdat(ProfileVarsComdat);
372 ValuesPtrExpr =
373 ConstantExpr::getBitCast(ValuesVar, llvm::Type::getInt8PtrTy(Ctx));
374 }
375 }
376
323377 // Create data variable.
324 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
325378 auto *Int16Ty = Type::getInt16Ty(Ctx);
326 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
379 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
327380 Type *DataTypes[] = {
328381 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
329382 #include "llvm/ProfileData/InstrProfData.inc"
366419 return CounterPtr;
367420 }
368421
369 static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
370 // Don't do this for Darwin. compiler-rt uses linker magic.
371 if (Triple(M.getTargetTriple()).isOSDarwin())
372 return false;
373
374 // Use linker script magic to get data/cnts/name start/end.
375 if (Triple(M.getTargetTriple()).isOSLinux() ||
376 Triple(M.getTargetTriple()).isOSFreeBSD() ||
377 Triple(M.getTargetTriple()).isPS4CPU())
378 return false;
379
380 return true;
422 void InstrProfiling::emitVNodes() {
423 if (!ValueProfileStaticAlloc)
424 return;
425
426 // For now only support this on platforms that do
427 // not require runtime registration to discover
428 // named section start/end.
429 if (needsRuntimeRegistrationOfSectionRange(*M))
430 return;
431
432 size_t TotalNS = 0;
433 for (auto &PD : ProfileDataMap) {
434 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
435 TotalNS += PD.second.NumValueSites[Kind];
436 }
437
438 if (!TotalNS)
439 return;
440
441 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
442 // Heuristic for small programs with very few total value sites.
443 // The default value of vp-counters-per-site is chosen based on
444 // the observation that large apps usually have a low percentage
445 // of value sites that actually have any profile data, and thus
446 // the average number of counters per site is low. For small
447 // apps with very few sites, this may not be true. Bump up the
448 // number of counters in this case.
449 if (NumCounters < 10)
450 NumCounters *= 2;
451
452 auto &Ctx = M->getContext();
453 Type *VNodeTypes[] = {
454 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
455 #include "llvm/ProfileData/InstrProfData.inc"
456 };
457 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
458
459 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
460 auto *VNodesVar = new GlobalVariable(
461 *M, VNodesTy, false, llvm::GlobalValue::PrivateLinkage,
462 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
463 VNodesVar->setSection(getInstrProfVNodesSectionName(isMachO()));
464 UsedVars.push_back(VNodesVar);
381465 }
382466
383467 void InstrProfiling::emitNameData() {
0 ;; Check that static counters are allocated for value profiler
1
2 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -vp-static-alloc=true -instrprof -S | FileCheck %s --check-prefix=STATIC
3 ; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC
4 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -vp-static-alloc=false -instrprof -S | FileCheck %s --check-prefix=DYN
5 ; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -vp-static-alloc=false -S | FileCheck %s --check-prefix=DYN
6
7
8 @__profn_foo = private constant [3 x i8] c"foo"
9
10 define i32 @foo(i32 ()* ) {
11 call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
12 %2 = ptrtoint i32 ()* %0 to i64
13 call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 12884901887, i64 %2, i32 0, i32 0)
14 %3 = tail call i32 %0()
15 ret i32 %3
16 }
17
18 ; Function Attrs: nounwind
19 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
20
21 ; Function Attrs: nounwind
22 declare void @llvm.instrprof.value.profile(i8*, i64, i64, i32, i32) #0
23
24 attributes #0 = { nounwind }
25
26 ; STATIC: @__profvp_foo
27 ; STATIC: @__llvm_prf_vnodes
28
29 ; DYN-NOT: @__profvp_foo
30 ; DYN-NOT: @__llvm_prf_vnodes