LLVM 19.0.0git
ModuleSummaryAnalysis.cpp
Go to the documentation of this file.
1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/StringRef.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Dominators.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalAlias.h"
39#include "llvm/IR/GlobalValue.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
51#include "llvm/Pass.h"
55#include <algorithm>
56#include <cassert>
57#include <cstdint>
58#include <vector>
59
60using namespace llvm;
61using namespace llvm::memprof;
62
63#define DEBUG_TYPE "module-summary-analysis"
64
65// Option to force edges cold which will block importing when the
66// -import-cold-multiplier is set to 0. Useful for debugging.
67namespace llvm {
70} // namespace llvm
71
73 "force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
74 cl::desc("Force all edges in the function summary to cold"),
77 "all-non-critical", "All non-critical edges."),
78 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
79
81 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
82 cl::desc("File to emit dot graph of new summary into"));
83
85
87
88// Walk through the operands of a given User via worklist iteration and populate
89// the set of GlobalValue references encountered. Invoked either on an
90// Instruction or a GlobalVariable (which walks its initializer).
91// Return true if any of the operands contains blockaddress. This is important
92// to know when computing summary for global var, because if global variable
93// references basic block address we can't import it separately from function
94// containing that basic block. For simplicity we currently don't import such
95// global vars at all. When importing function we aren't interested if any
96// instruction in it takes an address of any basic block, because instruction
97// can only take an address of basic block located in the same function.
98// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
99// local-linkage ifunc.
100static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
101 SetVector<ValueInfo, std::vector<ValueInfo>> &RefEdges,
103 bool &RefLocalLinkageIFunc) {
104 bool HasBlockAddress = false;
106 if (Visited.insert(CurUser).second)
107 Worklist.push_back(CurUser);
108
109 while (!Worklist.empty()) {
110 const User *U = Worklist.pop_back_val();
111 const auto *CB = dyn_cast<CallBase>(U);
112
113 for (const auto &OI : U->operands()) {
114 const User *Operand = dyn_cast<User>(OI);
115 if (!Operand)
116 continue;
117 if (isa<BlockAddress>(Operand)) {
118 HasBlockAddress = true;
119 continue;
120 }
121 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
122 // We have a reference to a global value. This should be added to
123 // the reference set unless it is a callee. Callees are handled
124 // specially by WriteFunction and are added to a separate list.
125 if (!(CB && CB->isCallee(&OI))) {
126 // If an ifunc has local linkage, do not add it into ref edges, and
127 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
128 // for import. An ifunc doesn't have summary and ThinLTO cannot
129 // promote it; importing the referencer may cause linkage errors.
130 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(GV);
131 GI && GI->hasLocalLinkage()) {
132 RefLocalLinkageIFunc = true;
133 continue;
134 }
135 RefEdges.insert(Index.getOrInsertValueInfo(GV));
136 }
137 continue;
138 }
139 if (Visited.insert(Operand).second)
140 Worklist.push_back(Operand);
141 }
142 }
143
144 const Instruction *I = dyn_cast<Instruction>(CurUser);
145 if (I) {
146 uint32_t ActualNumValueData = 0;
147 uint64_t TotalCount = 0;
148 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
149 // the instruction.
150 auto ValueDataArray =
152 ActualNumValueData, TotalCount);
153
154 if (ValueDataArray.get()) {
155 for (uint32_t j = 0; j < ActualNumValueData; j++) {
156 RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */
157 ValueDataArray[j].Value));
158 }
159 }
160 }
161 return HasBlockAddress;
162}
163
165 ProfileSummaryInfo *PSI) {
166 if (!PSI)
167 return CalleeInfo::HotnessType::Unknown;
168 if (PSI->isHotCount(ProfileCount))
169 return CalleeInfo::HotnessType::Hot;
170 if (PSI->isColdCount(ProfileCount))
171 return CalleeInfo::HotnessType::Cold;
172 return CalleeInfo::HotnessType::None;
173}
174
175static bool isNonRenamableLocal(const GlobalValue &GV) {
176 return GV.hasSection() && GV.hasLocalLinkage();
177}
178
179/// Determine whether this call has all constant integer arguments (excluding
180/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
181static void addVCallToSet(
183 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
184 &VCalls,
186 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
187 std::vector<uint64_t> Args;
188 // Start from the second argument to skip the "this" pointer.
189 for (auto &Arg : drop_begin(Call.CB.args())) {
190 auto *CI = dyn_cast<ConstantInt>(Arg);
191 if (!CI || CI->getBitWidth() > 64) {
192 VCalls.insert({Guid, Call.Offset});
193 return;
194 }
195 Args.push_back(CI->getZExtValue());
196 }
197 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
198}
199
200/// If this intrinsic call requires that we add information to the function
201/// summary, do so via the non-constant reference arguments.
203 const CallInst *CI,
204 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
205 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
206 &TypeTestAssumeVCalls,
207 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
208 &TypeCheckedLoadVCalls,
210 std::vector<FunctionSummary::ConstVCall>>
211 &TypeTestAssumeConstVCalls,
213 std::vector<FunctionSummary::ConstVCall>>
214 &TypeCheckedLoadConstVCalls,
215 DominatorTree &DT) {
216 switch (CI->getCalledFunction()->getIntrinsicID()) {
217 case Intrinsic::type_test:
218 case Intrinsic::public_type_test: {
219 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
220 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
221 if (!TypeId)
222 break;
223 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
224
225 // Produce a summary from type.test intrinsics. We only summarize type.test
226 // intrinsics that are used other than by an llvm.assume intrinsic.
227 // Intrinsics that are assumed are relevant only to the devirtualization
228 // pass, not the type test lowering pass.
229 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
230 return !isa<AssumeInst>(CIU.getUser());
231 });
232 if (HasNonAssumeUses)
233 TypeTests.insert(Guid);
234
237 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
238 for (auto &Call : DevirtCalls)
239 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
240 TypeTestAssumeConstVCalls);
241
242 break;
243 }
244
245 case Intrinsic::type_checked_load_relative:
246 case Intrinsic::type_checked_load: {
247 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
248 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
249 if (!TypeId)
250 break;
251 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
252
256 bool HasNonCallUses = false;
257 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
258 HasNonCallUses, CI, DT);
259 // Any non-call uses of the result of llvm.type.checked.load will
260 // prevent us from optimizing away the llvm.type.test.
261 if (HasNonCallUses)
262 TypeTests.insert(Guid);
263 for (auto &Call : DevirtCalls)
264 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
265 TypeCheckedLoadConstVCalls);
266
267 break;
268 }
269 default:
270 break;
271 }
272}
273
274static bool isNonVolatileLoad(const Instruction *I) {
275 if (const auto *LI = dyn_cast<LoadInst>(I))
276 return !LI->isVolatile();
277
278 return false;
279}
280
281static bool isNonVolatileStore(const Instruction *I) {
282 if (const auto *SI = dyn_cast<StoreInst>(I))
283 return !SI->isVolatile();
284
285 return false;
286}
287
288// Returns true if the function definition must be unreachable.
289//
290// Note if this helper function returns true, `F` is guaranteed
291// to be unreachable; if it returns false, `F` might still
292// be unreachable but not covered by this helper function.
294 // A function must be unreachable if its entry block ends with an
295 // 'unreachable'.
296 assert(!F.isDeclaration());
297 return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
298}
299
301 ModuleSummaryIndex &Index, const Module &M, const Function &F,
303 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
304 bool IsThinLTO,
305 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
306 // Summary not currently supported for anonymous functions, they should
307 // have been named.
308 assert(F.hasName());
309
310 unsigned NumInsts = 0;
311 // Map from callee ValueId to profile count. Used to accumulate profile
312 // counts for all static calls to a given callee.
314 std::vector<std::pair<ValueInfo, CalleeInfo>>>
315 CallGraphEdges;
316 SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges, LoadRefEdges,
317 StoreRefEdges;
320 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
322 std::vector<FunctionSummary::ConstVCall>>
323 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
324 ICallPromotionAnalysis ICallAnalysis;
326
327 // Add personality function, prefix data and prologue data to function's ref
328 // list.
329 bool HasLocalIFuncCallOrRef = false;
330 findRefEdges(Index, &F, RefEdges, Visited, HasLocalIFuncCallOrRef);
331 std::vector<const Instruction *> NonVolatileLoads;
332 std::vector<const Instruction *> NonVolatileStores;
333
334 std::vector<CallsiteInfo> Callsites;
335 std::vector<AllocInfo> Allocs;
336
337#ifndef NDEBUG
338 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
339#endif
340
341 bool HasInlineAsmMaybeReferencingInternal = false;
342 bool HasIndirBranchToBlockAddress = false;
343 bool HasUnknownCall = false;
344 bool MayThrow = false;
345 for (const BasicBlock &BB : F) {
346 // We don't allow inlining of function with indirect branch to blockaddress.
347 // If the blockaddress escapes the function, e.g., via a global variable,
348 // inlining may lead to an invalid cross-function reference. So we shouldn't
349 // import such function either.
350 if (BB.hasAddressTaken()) {
351 for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users())
352 if (!isa<CallBrInst>(*U)) {
353 HasIndirBranchToBlockAddress = true;
354 break;
355 }
356 }
357
358 for (const Instruction &I : BB) {
359 if (I.isDebugOrPseudoInst())
360 continue;
361 ++NumInsts;
362
363 // Regular LTO module doesn't participate in ThinLTO import,
364 // so no reference from it can be read/writeonly, since this
365 // would require importing variable as local copy
366 if (IsThinLTO) {
367 if (isNonVolatileLoad(&I)) {
368 // Postpone processing of non-volatile load instructions
369 // See comments below
370 Visited.insert(&I);
371 NonVolatileLoads.push_back(&I);
372 continue;
373 } else if (isNonVolatileStore(&I)) {
374 Visited.insert(&I);
375 NonVolatileStores.push_back(&I);
376 // All references from second operand of store (destination address)
377 // can be considered write-only if they're not referenced by any
378 // non-store instruction. References from first operand of store
379 // (stored value) can't be treated either as read- or as write-only
380 // so we add them to RefEdges as we do with all other instructions
381 // except non-volatile load.
382 Value *Stored = I.getOperand(0);
383 if (auto *GV = dyn_cast<GlobalValue>(Stored))
384 // findRefEdges will try to examine GV operands, so instead
385 // of calling it we should add GV to RefEdges directly.
386 RefEdges.insert(Index.getOrInsertValueInfo(GV));
387 else if (auto *U = dyn_cast<User>(Stored))
388 findRefEdges(Index, U, RefEdges, Visited, HasLocalIFuncCallOrRef);
389 continue;
390 }
391 }
392 findRefEdges(Index, &I, RefEdges, Visited, HasLocalIFuncCallOrRef);
393 const auto *CB = dyn_cast<CallBase>(&I);
394 if (!CB) {
395 if (I.mayThrow())
396 MayThrow = true;
397 continue;
398 }
399
400 const auto *CI = dyn_cast<CallInst>(&I);
401 // Since we don't know exactly which local values are referenced in inline
402 // assembly, conservatively mark the function as possibly referencing
403 // a local value from inline assembly to ensure we don't export a
404 // reference (which would require renaming and promotion of the
405 // referenced value).
406 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
407 HasInlineAsmMaybeReferencingInternal = true;
408
409 auto *CalledValue = CB->getCalledOperand();
410 auto *CalledFunction = CB->getCalledFunction();
411 if (CalledValue && !CalledFunction) {
412 CalledValue = CalledValue->stripPointerCasts();
413 // Stripping pointer casts can reveal a called function.
414 CalledFunction = dyn_cast<Function>(CalledValue);
415 }
416 // Check if this is an alias to a function. If so, get the
417 // called aliasee for the checks below.
418 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
419 assert(!CalledFunction && "Expected null called function in callsite for alias");
420 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
421 }
422 // Check if this is a direct call to a known function or a known
423 // intrinsic, or an indirect call with profile data.
424 if (CalledFunction) {
425 if (CI && CalledFunction->isIntrinsic()) {
427 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
428 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
429 continue;
430 }
431 // We should have named any anonymous globals
432 assert(CalledFunction->hasName());
433 auto ScaledCount = PSI->getProfileCount(*CB, BFI);
434 auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI)
435 : CalleeInfo::HotnessType::Unknown;
437 Hotness = CalleeInfo::HotnessType::Cold;
438
439 // Use the original CalledValue, in case it was an alias. We want
440 // to record the call edge to the alias in that case. Eventually
441 // an alias summary will be created to associate the alias and
442 // aliasee.
443 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
444 cast<GlobalValue>(CalledValue))];
445 ValueInfo.updateHotness(Hotness);
446 if (CB->isTailCall())
447 ValueInfo.setHasTailCall(true);
448 // Add the relative block frequency to CalleeInfo if there is no profile
449 // information.
450 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
451 uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
452 uint64_t EntryFreq = BFI->getEntryFreq().getFrequency();
453 ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
454 }
455 } else {
456 HasUnknownCall = true;
457 // If F is imported, a local linkage ifunc (e.g. target_clones on a
458 // static function) called by F will be cloned. Since summaries don't
459 // track ifunc, we do not know implementation functions referenced by
460 // the ifunc resolver need to be promoted in the exporter, and we will
461 // get linker errors due to cloned declarations for implementation
462 // functions. As a simple fix, just mark F as not eligible for import.
463 // Non-local ifunc is not cloned and does not have the issue.
464 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
465 if (GI->hasLocalLinkage())
466 HasLocalIFuncCallOrRef = true;
467 // Skip inline assembly calls.
468 if (CI && CI->isInlineAsm())
469 continue;
470 // Skip direct calls.
471 if (!CalledValue || isa<Constant>(CalledValue))
472 continue;
473
474 // Check if the instruction has a callees metadata. If so, add callees
475 // to CallGraphEdges to reflect the references from the metadata, and
476 // to enable importing for subsequent indirect call promotion and
477 // inlining.
478 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
479 for (const auto &Op : MD->operands()) {
480 Function *Callee = mdconst::extract_or_null<Function>(Op);
481 if (Callee)
482 CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
483 }
484 }
485
486 uint32_t NumVals, NumCandidates;
487 uint64_t TotalCount;
488 auto CandidateProfileData =
490 &I, NumVals, TotalCount, NumCandidates);
491 for (const auto &Candidate : CandidateProfileData)
492 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
493 .updateHotness(getHotness(Candidate.Count, PSI));
494 }
495
496 // Summarize memprof related metadata. This is only needed for ThinLTO.
497 if (!IsThinLTO)
498 continue;
499
500 // TODO: Skip indirect calls for now. Need to handle these better, likely
501 // by creating multiple Callsites, one per target, then speculatively
502 // devirtualize while applying clone info in the ThinLTO backends. This
503 // will also be important because we will have a different set of clone
504 // versions per target. This handling needs to match that in the ThinLTO
505 // backend so we handle things consistently for matching of callsite
506 // summaries to instructions.
507 if (!CalledFunction)
508 continue;
509
510 // Ensure we keep this analysis in sync with the handling in the ThinLTO
511 // backend (see MemProfContextDisambiguation::applyImport). Save this call
512 // so that we can skip it in checking the reverse case later.
514#ifndef NDEBUG
515 CallsThatMayHaveMemprofSummary.insert(CB);
516#endif
517
518 // Compute the list of stack ids first (so we can trim them from the stack
519 // ids on any MIBs).
521 I.getMetadata(LLVMContext::MD_callsite));
522 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
523 if (MemProfMD) {
524 std::vector<MIBInfo> MIBs;
525 for (auto &MDOp : MemProfMD->operands()) {
526 auto *MIBMD = cast<const MDNode>(MDOp);
529 SmallVector<unsigned> StackIdIndices;
531 // Collapse out any on the allocation call (inlining).
532 for (auto ContextIter =
533 StackContext.beginAfterSharedPrefix(InstCallsite);
534 ContextIter != StackContext.end(); ++ContextIter) {
535 unsigned StackIdIdx = Index.addOrGetStackIdIndex(*ContextIter);
536 // If this is a direct recursion, simply skip the duplicate
537 // entries. If this is mutual recursion, handling is left to
538 // the LTO link analysis client.
539 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
540 StackIdIndices.push_back(StackIdIdx);
541 }
542 MIBs.push_back(
543 MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
544 }
545 Allocs.push_back(AllocInfo(std::move(MIBs)));
546 } else if (!InstCallsite.empty()) {
547 SmallVector<unsigned> StackIdIndices;
548 for (auto StackId : InstCallsite)
549 StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
550 // Use the original CalledValue, in case it was an alias. We want
551 // to record the call edge to the alias in that case. Eventually
552 // an alias summary will be created to associate the alias and
553 // aliasee.
554 auto CalleeValueInfo =
555 Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
556 Callsites.push_back({CalleeValueInfo, StackIdIndices});
557 }
558 }
559 }
560
562 Index.addBlockCount(F.size());
563
564 std::vector<ValueInfo> Refs;
565 if (IsThinLTO) {
566 auto AddRefEdges = [&](const std::vector<const Instruction *> &Instrs,
569 for (const auto *I : Instrs) {
570 Cache.erase(I);
571 findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
572 }
573 };
574
575 // By now we processed all instructions in a function, except
576 // non-volatile loads and non-volatile value stores. Let's find
577 // ref edges for both of instruction sets
578 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
579 // We can add some values to the Visited set when processing load
580 // instructions which are also used by stores in NonVolatileStores.
581 // For example this can happen if we have following code:
582 //
583 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
584 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
585 //
586 // After processing loads we'll add bitcast to the Visited set, and if
587 // we use the same set while processing stores, we'll never see store
588 // to @bar and @bar will be mistakenly treated as readonly.
590 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
591
592 // If both load and store instruction reference the same variable
593 // we won't be able to optimize it. Add all such reference edges
594 // to RefEdges set.
595 for (const auto &VI : StoreRefEdges)
596 if (LoadRefEdges.remove(VI))
597 RefEdges.insert(VI);
598
599 unsigned RefCnt = RefEdges.size();
600 // All new reference edges inserted in two loops below are either
601 // read or write only. They will be grouped in the end of RefEdges
602 // vector, so we can use a single integer value to identify them.
603 for (const auto &VI : LoadRefEdges)
604 RefEdges.insert(VI);
605
606 unsigned FirstWORef = RefEdges.size();
607 for (const auto &VI : StoreRefEdges)
608 RefEdges.insert(VI);
609
610 Refs = RefEdges.takeVector();
611 for (; RefCnt < FirstWORef; ++RefCnt)
612 Refs[RefCnt].setReadOnly();
613
614 for (; RefCnt < Refs.size(); ++RefCnt)
615 Refs[RefCnt].setWriteOnly();
616 } else {
617 Refs = RefEdges.takeVector();
618 }
619 // Explicit add hot edges to enforce importing for designated GUIDs for
620 // sample PGO, to enable the same inlines as the profiled optimized binary.
621 for (auto &I : F.getImportGUIDs())
622 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
624 ? CalleeInfo::HotnessType::Cold
625 : CalleeInfo::HotnessType::Critical);
626
627#ifndef NDEBUG
628 // Make sure that all calls we decided could not have memprof summaries get a
629 // false value for mayHaveMemprofSummary, to ensure that this handling remains
630 // in sync with the ThinLTO backend handling.
631 if (IsThinLTO) {
632 for (const BasicBlock &BB : F) {
633 for (const Instruction &I : BB) {
634 const auto *CB = dyn_cast<CallBase>(&I);
635 if (!CB)
636 continue;
637 // We already checked these above.
638 if (CallsThatMayHaveMemprofSummary.count(CB))
639 continue;
641 }
642 }
643 }
644#endif
645
646 bool NonRenamableLocal = isNonRenamableLocal(F);
647 bool NotEligibleForImport =
648 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
649 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
651 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
652 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
653 GlobalValueSummary::ImportKind::Definition);
655 F.doesNotAccessMemory(), F.onlyReadsMemory() && !F.doesNotAccessMemory(),
656 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
657 // FIXME: refactor this to use the same code that inliner is using.
658 // Don't try to import functions with noinline attribute.
659 F.getAttributes().hasFnAttr(Attribute::NoInline),
660 F.hasFnAttribute(Attribute::AlwaysInline),
661 F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
663 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
664 if (auto *SSI = GetSSICallback(F))
665 ParamAccesses = SSI->getParamAccesses(Index);
666 auto FuncSummary = std::make_unique<FunctionSummary>(
667 Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
668 CallGraphEdges.takeVector(), TypeTests.takeVector(),
669 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
670 TypeTestAssumeConstVCalls.takeVector(),
671 TypeCheckedLoadConstVCalls.takeVector(), std::move(ParamAccesses),
672 std::move(Callsites), std::move(Allocs));
673 if (NonRenamableLocal)
674 CantBePromoted.insert(F.getGUID());
675 Index.addGlobalValueSummary(F, std::move(FuncSummary));
676}
677
678/// Find function pointers referenced within the given vtable initializer
679/// (or subset of an initializer) \p I. The starting offset of \p I within
680/// the vtable initializer is \p StartingOffset. Any discovered function
681/// pointers are added to \p VTableFuncs along with their cumulative offset
682/// within the initializer.
683static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
685 VTableFuncList &VTableFuncs,
686 const GlobalVariable &OrigGV) {
687 // First check if this is a function pointer.
688 if (I->getType()->isPointerTy()) {
689 auto C = I->stripPointerCasts();
690 auto A = dyn_cast<GlobalAlias>(C);
691 if (isa<Function>(C) || (A && isa<Function>(A->getAliasee()))) {
692 auto GV = dyn_cast<GlobalValue>(C);
693 assert(GV);
694 // We can disregard __cxa_pure_virtual as a possible call target, as
695 // calls to pure virtuals are UB.
696 if (GV && GV->getName() != "__cxa_pure_virtual")
697 VTableFuncs.push_back({Index.getOrInsertValueInfo(GV), StartingOffset});
698 return;
699 }
700 }
701
702 // Walk through the elements in the constant struct or array and recursively
703 // look for virtual function pointers.
704 const DataLayout &DL = M.getDataLayout();
705 if (auto *C = dyn_cast<ConstantStruct>(I)) {
706 StructType *STy = dyn_cast<StructType>(C->getType());
707 assert(STy);
708 const StructLayout *SL = DL.getStructLayout(C->getType());
709
710 for (auto EI : llvm::enumerate(STy->elements())) {
711 auto Offset = SL->getElementOffset(EI.index());
712 unsigned Op = SL->getElementContainingOffset(Offset);
713 findFuncPointers(cast<Constant>(I->getOperand(Op)),
714 StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
715 }
716 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
717 ArrayType *ATy = C->getType();
718 Type *EltTy = ATy->getElementType();
719 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
720 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
721 findFuncPointers(cast<Constant>(I->getOperand(i)),
722 StartingOffset + i * EltSize, M, Index, VTableFuncs,
723 OrigGV);
724 }
725 } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
726 // For relative vtables, the next sub-component should be a trunc.
727 if (CE->getOpcode() != Instruction::Trunc ||
728 !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
729 return;
730
731 // If this constant can be reduced to the offset between a function and a
732 // global, then we know this is a valid virtual function if the RHS is the
733 // original vtable we're scanning through.
734 if (CE->getOpcode() == Instruction::Sub) {
736 APSInt LHSOffset, RHSOffset;
737 if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
738 IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
739 RHS == &OrigGV &&
740
741 // For relative vtables, this component should point to the callable
742 // function without any offsets.
743 LHSOffset == 0 &&
744
745 // Also, the RHS should always point to somewhere within the vtable.
746 RHSOffset <=
747 static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
748 findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
749 }
750 }
751 }
752}
753
754// Identify the function pointers referenced by vtable definition \p V.
756 const GlobalVariable &V, const Module &M,
757 VTableFuncList &VTableFuncs) {
758 if (!V.isConstant())
759 return;
760
761 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
762 VTableFuncs, V);
763
764#ifndef NDEBUG
765 // Validate that the VTableFuncs list is ordered by offset.
766 uint64_t PrevOffset = 0;
767 for (auto &P : VTableFuncs) {
768 // The findVFuncPointers traversal should have encountered the
769 // functions in offset order. We need to use ">=" since PrevOffset
770 // starts at 0.
771 assert(P.VTableOffset >= PrevOffset);
772 PrevOffset = P.VTableOffset;
773 }
774#endif
775}
776
777/// Record vtable definition \p V for each type metadata it references.
778static void
780 const GlobalVariable &V,
782 for (MDNode *Type : Types) {
783 auto TypeID = Type->getOperand(1).get();
784
786 cast<ConstantInt>(
787 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
788 ->getZExtValue();
789
790 if (auto *TypeId = dyn_cast<MDString>(TypeID))
791 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
792 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
793 }
794}
795
797 const GlobalVariable &V,
798 DenseSet<GlobalValue::GUID> &CantBePromoted,
799 const Module &M,
803 bool RefLocalIFunc = false;
804 bool HasBlockAddress =
805 findRefEdges(Index, &V, RefEdges, Visited, RefLocalIFunc);
806 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
807 bool NonRenamableLocal = isNonRenamableLocal(V);
809 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
810 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
812
813 VTableFuncList VTableFuncs;
814 // If splitting is not enabled, then we compute the summary information
815 // necessary for index-based whole program devirtualization.
816 if (!Index.enableSplitLTOUnit()) {
817 Types.clear();
818 V.getMetadata(LLVMContext::MD_type, Types);
819 if (!Types.empty()) {
820 // Identify the function pointers referenced by this vtable definition.
821 computeVTableFuncs(Index, V, M, VTableFuncs);
822
823 // Record this vtable definition for each type metadata it references.
825 }
826 }
827
828 // Don't mark variables we won't be able to internalize as read/write-only.
829 bool CanBeInternalized =
830 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
831 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
832 bool Constant = V.isConstant();
833 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
834 Constant ? false : CanBeInternalized,
835 Constant, V.getVCallVisibility());
836 auto GVarSummary = std::make_unique<GlobalVarSummary>(Flags, VarFlags,
837 RefEdges.takeVector());
838 if (NonRenamableLocal)
839 CantBePromoted.insert(V.getGUID());
840 if (NotEligibleForImport)
841 GVarSummary->setNotEligibleToImport();
842 if (!VTableFuncs.empty())
843 GVarSummary->setVTableFuncs(VTableFuncs);
844 Index.addGlobalValueSummary(V, std::move(GVarSummary));
845}
846
848 DenseSet<GlobalValue::GUID> &CantBePromoted) {
849 // Skip summary for indirect function aliases as summary for aliasee will not
850 // be emitted.
851 const GlobalObject *Aliasee = A.getAliaseeObject();
852 if (isa<GlobalIFunc>(Aliasee))
853 return;
854 bool NonRenamableLocal = isNonRenamableLocal(A);
856 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
857 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
859 auto AS = std::make_unique<AliasSummary>(Flags);
860 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
861 assert(AliaseeVI && "Alias expects aliasee summary to be available");
862 assert(AliaseeVI.getSummaryList().size() == 1 &&
863 "Expected a single entry per aliasee in per-module index");
864 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
865 if (NonRenamableLocal)
866 CantBePromoted.insert(A.getGUID());
867 Index.addGlobalValueSummary(A, std::move(AS));
868}
869
870// Set LiveRoot flag on entries matching the given value name.
872 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
873 for (const auto &Summary : VI.getSummaryList())
874 Summary->setLive(true);
875}
876
878 const Module &M,
879 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
881 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
882 assert(PSI);
883 bool EnableSplitLTOUnit = false;
884 bool UnifiedLTO = false;
885 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
886 M.getModuleFlag("EnableSplitLTOUnit")))
887 EnableSplitLTOUnit = MD->getZExtValue();
888 if (auto *MD =
889 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("UnifiedLTO")))
890 UnifiedLTO = MD->getZExtValue();
891 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
892
893 // Identify the local values in the llvm.used and llvm.compiler.used sets,
894 // which should not be exported as they would then require renaming and
895 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
896 // here because we use this information to mark functions containing inline
897 // assembly calls as not importable.
900 // First collect those in the llvm.used set.
901 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/false);
902 // Next collect those in the llvm.compiler.used set.
903 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/true);
904 DenseSet<GlobalValue::GUID> CantBePromoted;
905 for (auto *V : Used) {
906 if (V->hasLocalLinkage()) {
907 LocalsUsed.insert(V);
908 CantBePromoted.insert(V->getGUID());
909 }
910 }
911
912 bool HasLocalInlineAsmSymbol = false;
913 if (!M.getModuleInlineAsm().empty()) {
914 // Collect the local values defined by module level asm, and set up
915 // summaries for these symbols so that they can be marked as NoRename,
916 // to prevent export of any use of them in regular IR that would require
917 // renaming within the module level asm. Note we don't need to create a
918 // summary for weak or global defs, as they don't need to be flagged as
919 // NoRename, and defs in module level asm can't be imported anyway.
920 // Also, any values used but not defined within module level asm should
921 // be listed on the llvm.used or llvm.compiler.used global and marked as
922 // referenced from there.
925 // Symbols not marked as Weak or Global are local definitions.
928 return;
929 HasLocalInlineAsmSymbol = true;
930 GlobalValue *GV = M.getNamedValue(Name);
931 if (!GV)
932 return;
933 assert(GV->isDeclaration() && "Def in module asm already has definition");
936 /* NotEligibleToImport = */ true,
937 /* Live = */ true,
938 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
940 CantBePromoted.insert(GV->getGUID());
941 // Create the appropriate summary type.
942 if (Function *F = dyn_cast<Function>(GV)) {
943 std::unique_ptr<FunctionSummary> Summary =
944 std::make_unique<FunctionSummary>(
945 GVFlags, /*InstCount=*/0,
947 F->hasFnAttribute(Attribute::ReadNone),
948 F->hasFnAttribute(Attribute::ReadOnly),
949 F->hasFnAttribute(Attribute::NoRecurse),
950 F->returnDoesNotAlias(),
951 /* NoInline = */ false,
952 F->hasFnAttribute(Attribute::AlwaysInline),
953 F->hasFnAttribute(Attribute::NoUnwind),
954 /* MayThrow */ true,
955 /* HasUnknownCall */ true,
956 /* MustBeUnreachable */ false},
957 /*EntryCount=*/0, ArrayRef<ValueInfo>{},
966 Index.addGlobalValueSummary(*GV, std::move(Summary));
967 } else {
968 std::unique_ptr<GlobalVarSummary> Summary =
969 std::make_unique<GlobalVarSummary>(
970 GVFlags,
972 false, false, cast<GlobalVariable>(GV)->isConstant(),
975 Index.addGlobalValueSummary(*GV, std::move(Summary));
976 }
977 });
978 }
979
980 bool IsThinLTO = true;
981 if (auto *MD =
982 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
983 IsThinLTO = MD->getZExtValue();
984
985 // Compute summaries for all functions defined in module, and save in the
986 // index.
987 for (const auto &F : M) {
988 if (F.isDeclaration())
989 continue;
990
991 DominatorTree DT(const_cast<Function &>(F));
992 BlockFrequencyInfo *BFI = nullptr;
993 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
994 if (GetBFICallback)
995 BFI = GetBFICallback(F);
996 else if (F.hasProfileData()) {
997 LoopInfo LI{DT};
998 BranchProbabilityInfo BPI{F, LI};
999 BFIPtr = std::make_unique<BlockFrequencyInfo>(F, BPI, LI);
1000 BFI = BFIPtr.get();
1001 }
1002
1003 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
1004 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1005 CantBePromoted, IsThinLTO, GetSSICallback);
1006 }
1007
1008 // Compute summaries for all variables defined in module, and save in the
1009 // index.
1011 for (const GlobalVariable &G : M.globals()) {
1012 if (G.isDeclaration())
1013 continue;
1014 computeVariableSummary(Index, G, CantBePromoted, M, Types);
1015 }
1016
1017 // Compute summaries for all aliases defined in module, and save in the
1018 // index.
1019 for (const GlobalAlias &A : M.aliases())
1020 computeAliasSummary(Index, A, CantBePromoted);
1021
1022 // Iterate through ifuncs, set their resolvers all alive.
1023 for (const GlobalIFunc &I : M.ifuncs()) {
1024 I.applyAlongResolverPath([&Index](const GlobalValue &GV) {
1025 Index.getGlobalValueSummary(GV)->setLive(true);
1026 });
1027 }
1028
1029 for (auto *V : LocalsUsed) {
1030 auto *Summary = Index.getGlobalValueSummary(*V);
1031 assert(Summary && "Missing summary for global value");
1032 Summary->setNotEligibleToImport();
1033 }
1034
1035 // The linker doesn't know about these LLVM produced values, so we need
1036 // to flag them as live in the index to ensure index-based dead value
1037 // analysis treats them as live roots of the analysis.
1038 setLiveRoot(Index, "llvm.used");
1039 setLiveRoot(Index, "llvm.compiler.used");
1040 setLiveRoot(Index, "llvm.global_ctors");
1041 setLiveRoot(Index, "llvm.global_dtors");
1042 setLiveRoot(Index, "llvm.global.annotations");
1043
1044 for (auto &GlobalList : Index) {
1045 // Ignore entries for references that are undefined in the current module.
1046 if (GlobalList.second.SummaryList.empty())
1047 continue;
1048
1049 assert(GlobalList.second.SummaryList.size() == 1 &&
1050 "Expected module's index to have one summary per GUID");
1051 auto &Summary = GlobalList.second.SummaryList[0];
1052 if (!IsThinLTO) {
1053 Summary->setNotEligibleToImport();
1054 continue;
1055 }
1056
1057 bool AllRefsCanBeExternallyReferenced =
1058 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
1059 return !CantBePromoted.count(VI.getGUID());
1060 });
1061 if (!AllRefsCanBeExternallyReferenced) {
1062 Summary->setNotEligibleToImport();
1063 continue;
1064 }
1065
1066 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
1067 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1068 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
1069 return !CantBePromoted.count(Edge.first.getGUID());
1070 });
1071 if (!AllCallsCanBeExternallyReferenced)
1072 Summary->setNotEligibleToImport();
1073 }
1074 }
1075
1076 if (!ModuleSummaryDotFile.empty()) {
1077 std::error_code EC;
1078 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_None);
1079 if (EC)
1080 report_fatal_error(Twine("Failed to open dot file ") +
1081 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1082 Index.exportToDot(OSDot, {});
1083 }
1084
1085 return Index;
1086}
1087
1088AnalysisKey ModuleSummaryIndexAnalysis::Key;
1089
1093 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1094 bool NeedSSI = needsParamAccessSummary(M);
1096 M,
1097 [&FAM](const Function &F) {
1099 *const_cast<Function *>(&F));
1100 },
1101 &PSI,
1102 [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1103 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1104 const_cast<Function &>(F))
1105 : nullptr;
1106 });
1107}
1108
1110
1112 "Module Summary Analysis", false, true)
1118
1120 return new ModuleSummaryIndexWrapperPass();
1121}
1122
1124 : ModulePass(ID) {
1126}
1127
1129 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1130 bool NeedSSI = needsParamAccessSummary(M);
1132 M,
1133 [this](const Function &F) {
1134 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1135 *const_cast<Function *>(&F))
1136 .getBFI());
1137 },
1138 PSI,
1139 [&](const Function &F) -> const StackSafetyInfo * {
1140 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1141 const_cast<Function &>(F))
1142 .getResult()
1143 : nullptr;
1144 }));
1145 return false;
1146}
1147
1149 Index.reset();
1150 return false;
1151}
1152
1154 AU.setPreservesAll();
1158}
1159
1161
1167}
1168
1170 AnalysisUsage &AU) const {
1171 AU.setPreservesAll();
1172}
1173
1175 const ModuleSummaryIndex *Index) {
1177}
1178
1180 "Module summary info", false, true)
1181
1183 if (!CB)
1184 return false;
1185 if (CB->isDebugOrPseudoInst())
1186 return false;
1187 auto *CI = dyn_cast<CallInst>(CB);
1188 auto *CalledValue = CB->getCalledOperand();
1189 auto *CalledFunction = CB->getCalledFunction();
1190 if (CalledValue && !CalledFunction) {
1191 CalledValue = CalledValue->stripPointerCasts();
1192 // Stripping pointer casts can reveal a called function.
1193 CalledFunction = dyn_cast<Function>(CalledValue);
1194 }
1195 // Check if this is an alias to a function. If so, get the
1196 // called aliasee for the checks below.
1197 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
1198 assert(!CalledFunction &&
1199 "Expected null called function in callsite for alias");
1200 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
1201 }
1202 // Check if this is a direct call to a known function or a known
1203 // intrinsic, or an indirect call with profile data.
1204 if (CalledFunction) {
1205 if (CI && CalledFunction->isIntrinsic())
1206 return false;
1207 } else {
1208 // TODO: For now skip indirect calls. See comments in
1209 // computeFunctionSummary for what is needed to handle this.
1210 return false;
1211 }
1212 return true;
1213}
aarch64 promote const
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file contains the simple types necessary to represent the attributes associated with functions a...
basic Basic Alias true
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:693
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
iv users
Definition: IVUsers.cpp:48
Interface to identify indirect call promotion candidates.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &VCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &ConstVCalls)
Determine whether this call has all constant integer arguments (excluding "this") and summarize it to...
cl::opt< unsigned > MaxNumVTableAnnotations
static void computeVTableFuncs(ModuleSummaryIndex &Index, const GlobalVariable &V, const Module &M, VTableFuncList &VTableFuncs)
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet< GlobalValue::GUID > &CantBePromoted)
static void findFuncPointers(const Constant *I, uint64_t StartingOffset, const Module &M, ModuleSummaryIndex &Index, VTableFuncList &VTableFuncs, const GlobalVariable &OrigGV)
Find function pointers referenced within the given vtable initializer (or subset of an initializer) I...
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet< GlobalValue::GUID > &CantBePromoted, const Module &M, SmallVectorImpl< MDNode * > &Types)
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name)
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount, ProfileSummaryInfo *PSI)
static bool isNonVolatileLoad(const Instruction *I)
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize
static bool isNonRenamableLocal(const GlobalValue &GV)
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, bool HasLocalsInUsedOrAsm, DenseSet< GlobalValue::GUID > &CantBePromoted, bool IsThinLTO, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback)
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector< ValueInfo, std::vector< ValueInfo > > &RefEdges, SmallPtrSet< const User *, 8 > &Visited, bool &RefLocalLinkageIFunc)
static cl::opt< FunctionSummary::ForceSummaryHotnessType, true > FSEC("force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold), cl::desc("Force all edges in the function summary to cold"), cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."), clEnumValN(FunctionSummary::FSHT_AllNonCritical, "all-non-critical", "All non-critical edges."), clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")))
static bool mustBeUnreachableFunction(const Function &F)
static bool isNonVolatileStore(const Instruction *I)
module summary analysis
static cl::opt< std::string > ModuleSummaryDotFile("module-summary-dot-file", cl::Hidden, cl::value_desc("filename"), cl::desc("File to emit dot graph of new summary into"))
static void addIntrinsicToSummary(const CallInst *CI, SetVector< GlobalValue::GUID, std::vector< GlobalValue::GUID > > &TypeTests, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeTestAssumeVCalls, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeCheckedLoadVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeTestAssumeConstVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeCheckedLoadConstVCalls, DominatorTree &DT)
If this intrinsic call requires that we add information to the function summary, do so via the non-co...
static void recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, const GlobalVariable &V, SmallVectorImpl< MDNode * > &Types)
Record vtable definition V for each type metadata it references.
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This defines the Use class.
Value * RHS
Value * LHS
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:321
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:473
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Definition: Constants.cpp:1846
Analysis pass which computes BlockFrequencyInfo.
Legacy analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1494
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1742
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1687
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
ForceSummaryHotnessType
Types for -force-summary-edges-cold debugging option.
Class to represent profile counts.
Definition: Function.h:279
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:232
bool isDSOLocal() const
Definition: GlobalValue.h:304
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:286
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: Globals.cpp:75
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:594
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:66
bool hasSection() const
Definition: GlobalValue.h:289
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
Definition: Globals.cpp:405
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:58
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
ArrayRef< InstrProfValueData > getPromotionCandidatesForInstruction(const Instruction *I, uint32_t &NumVals, uint64_t &TotalCount, uint32_t &NumCandidates)
Returns reference to array of InstrProfValueData for the given instruction I.
Legacy wrapper pass to provide the ModuleSummaryIndex object.
ImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index=nullptr)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:282
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:631
Metadata node.
Definition: Metadata.h:1067
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType takeVector()
Clear the MapVector and return the underlying vector.
Definition: MapVector.h:55
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
Result run(Module &M, ModuleAnalysisManager &AM)
Legacy wrapper pass to provide the ModuleSummaryIndex object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static void CollectAsmSymbols(const Module &M, function_ref< void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol)
Parse inline ASM and collect the symbols that are defined or referenced in the current module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
A vector that has set insertion semantics.
Definition: SetVector.h:57
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:188
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StackSafetyInfo wrapper for the new pass manager.
StackSafetyInfo wrapper for the legacy pass manager.
Interface to access stack safety analysis results for single function.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:622
unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
Definition: DataLayout.cpp:92
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
ArrayRef< Type * > elements() const
Definition: DerivedTypes.h:333
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< use_iterator > uses()
Definition: Value.h:376
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator end() const
CallStackIterator beginAfterSharedPrefix(CallStack &Other)
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:718
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:456
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1378
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2406
bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, const DataLayout &DL, DSOLocalEquivalent **DSOEquiv=nullptr)
If this constant is a constant offset from a global, return the global and the constant.
FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold
bool needsParamAccessSummary(const Module &M)
ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &)
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
ModulePass * createModuleSummaryIndexWrapperPass()
ImmutablePass * createImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index)
void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &)
bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)
Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...
Definition: Module.cpp:843
Summary of memprof metadata on allocations.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:26
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
An "identifier" for a virtual function.
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
Summary of a single MIB in a memprof metadata on allocations.
Struct that holds a reference to a particular GUID in a global value summary.