llvm.org GIT mirror llvm / e7e4f1f
[ThinLTO] Consolidate cache key computation between new/old LTO APIs Summary: The old legacy LTO API had a separate cache key computation, which was a subset of the cache key computation in the new LTO API (from what I can tell this is largely just because certain features such as CFI, dsoLocal, etc are only utilized via the new LTO API). However, having separate computations is unnecessary (much of the code is duplicated), and can lead to bugs when adding new optimizations if both cache computation algorithms aren't updated properly - it's much easier to maintain if we have a single facility. This patch refactors the old LTO API code to use the cache key computation from the new LTO API. To do this, we set up an lto::Config object and fill in the fields that the old LTO was hashing (the others will just use the defaults). There are two notable changes: - I added a Freestanding flag to the LTO Config. Currently this is only used by the legacy LTO API. In the patch that added it (D30791) I had asked about adding it to the new LTO API, but it looks like that was not addressed. This should probably be discussed as a follow up to this change, as it is orthogonal. - The legacy LTO API had some code that was hashing the GUID of all preserved symbols defined in the module. I looked back at the history of this (which was added with the original hashing in the legacy LTO API in D18494), and there is a comment in the review thread that it was added in preparation for future internalization. We now do the internalization of course, and that is handled in the new LTO API cache key computation by hashing the recorded linkage type of all defined globals. Therefore I didn't try to move over and keep the preserved symbols handling. Reviewers: steven_wu, pcc Subscribers: mehdi_amini, inglorion, eraman, dexonsmith, dang, llvm-commits Differential Revision: https://reviews.llvm.org/D54635 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@347592 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson 9 months ago
4 changed file(s) with 40 addition(s) and 92 deletion(s). Raw diff Collapse all Expand all
4747
4848 /// Use the new pass manager
4949 bool UseNewPM = false;
50
51 /// Flag to indicate that the optimizer should not assume builtins are present
52 /// on the target.
53 bool Freestanding = false;
5054
5155 /// Disable entirely the optimizer, including importing for ThinLTO
5256 bool CodeGenOnly = false;
5858 void thinLTOInternalizeAndPromoteInIndex(
5959 ModuleSummaryIndex &Index,
6060 function_ref isExported);
61
62 /// Computes a unique hash for the Module considering the current list of
63 /// export/import and other global analysis results.
64 /// The hash is produced in \p Key.
65 void computeLTOCacheKey(
66 SmallString<40> &Key, const lto::Config &Conf,
67 const ModuleSummaryIndex &Index, StringRef ModuleID,
68 const FunctionImporter::ImportMapTy &ImportList,
69 const FunctionImporter::ExportSetTy &ExportList,
70 const std::map &ResolvedODR,
71 const GVSummaryMapTy &DefinedGlobals,
72 const std::set &CfiFunctionDefs = {},
73 const std::set &CfiFunctionDecls = {});
6174
6275 namespace lto {
6376
6060 "enable-lto-internalization", cl::init(true), cl::Hidden,
6161 cl::desc("Enable global value internalization in LTO"));
6262
63 // Returns a unique hash for the Module considering the current list of
63 // Computes a unique hash for the Module considering the current list of
6464 // export/import and other global analysis results.
6565 // The hash is produced in \p Key.
66 static void computeCacheKey(
66 void llvm::computeLTOCacheKey(
6767 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
6868 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
6969 const FunctionImporter::ExportSetTy &ExportList,
131131 AddUnsigned(Conf.CGFileType);
132132 AddUnsigned(Conf.OptLevel);
133133 AddUnsigned(Conf.UseNewPM);
134 AddUnsigned(Conf.Freestanding);
134135 AddString(Conf.OptPipeline);
135136 AddString(Conf.AAPipeline);
136137 AddString(Conf.OverrideTriple);
982983
983984 SmallString<40> Key;
984985 // The module may be cached, this helps handling it.
985 computeCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList,
986 ResolvedODR, DefinedGlobals, CfiFunctionDefs,
987 CfiFunctionDecls);
986 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
987 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
988 CfiFunctionDecls);
988989 if (AddStreamFn CacheAddStream = Cache(Task, Key))
989990 return RunThinBackend(CacheAddStream);
990991
297297 const FunctionImporter::ImportMapTy &ImportList,
298298 const FunctionImporter::ExportSetTy &ExportList,
299299 const std::map &ResolvedODR,
300 const GVSummaryMapTy &DefinedGVSummaries,
301 const DenseSet &PreservedSymbols, unsigned OptLevel,
300 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
302301 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
303302 if (CachePath.empty())
304303 return;
307306 // The module does not have an entry, it can't have a hash at all
308307 return;
309308
310 // Compute the unique hash for this entry
311 // This is based on the current compiler version, the module itself, the
312 // export list, the hash for every single module in the import list, the
313 // list of ResolvedODR for the module, and the list of preserved symbols.
314
315 // Include the hash for the current module
316 auto ModHash = Index.getModuleHash(ModuleID);
317
318 if (all_of(ModHash, [](uint32_t V) { return V == 0; }))
309 if (all_of(Index.getModuleHash(ModuleID),
310 [](uint32_t V) { return V == 0; }))
319311 // No hash entry, no caching!
320312 return;
321313
322 SHA1 Hasher;
323
324 // Include the parts of the LTO configuration that affect code generation.
325 auto AddString = [&](StringRef Str) {
326 Hasher.update(Str);
327 Hasher.update(ArrayRef{0});
328 };
329 auto AddUnsigned = [&](unsigned I) {
330 uint8_t Data[4];
331 Data[0] = I;
332 Data[1] = I >> 8;
333 Data[2] = I >> 16;
334 Data[3] = I >> 24;
335 Hasher.update(ArrayRef{Data, 4});
336 };
337
338 // Start with the compiler revision
339 Hasher.update(LLVM_VERSION_STRING);
340 #ifdef LLVM_REVISION
341 Hasher.update(LLVM_REVISION);
342 #endif
343
344 // Hash the optimization level and the target machine settings.
345 AddString(TMBuilder.MCpu);
346 // FIXME: Hash more of Options. For now all clients initialize Options from
347 // command-line flags (which is unsupported in production), but may set
348 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
349 // DataSections and DebuggerTuning via command line flags.
350 AddUnsigned(TMBuilder.Options.RelaxELFRelocations);
351 AddUnsigned(TMBuilder.Options.FunctionSections);
352 AddUnsigned(TMBuilder.Options.DataSections);
353 AddUnsigned((unsigned)TMBuilder.Options.DebuggerTuning);
354 AddString(TMBuilder.MAttr);
355 if (TMBuilder.RelocModel)
356 AddUnsigned(*TMBuilder.RelocModel);
357 AddUnsigned(TMBuilder.CGOptLevel);
358 AddUnsigned(OptLevel);
359 AddUnsigned(Freestanding);
360
361 Hasher.update(ArrayRef((uint8_t *)&ModHash[0], sizeof(ModHash)));
362 for (auto F : ExportList)
363 // The export list can impact the internalization, be conservative here
364 Hasher.update(ArrayRef((uint8_t *)&F, sizeof(F)));
365
366 // Include the hash for every module we import functions from
367 for (auto &Entry : ImportList) {
368 auto ModHash = Index.getModuleHash(Entry.first());
369 Hasher.update(ArrayRef((uint8_t *)&ModHash[0], sizeof(ModHash)));
370 for (auto Guid : Entry.second)
371 if (auto *GVS = dyn_cast(
372 Index.getGlobalValueSummary(Guid, false)))
373 AddUnsigned(GVS->isReadOnly());
374 }
375
376 // Include the hash for the resolved ODR.
377 for (auto &Entry : ResolvedODR) {
378 Hasher.update(ArrayRef((const uint8_t *)&Entry.first,
379 sizeof(GlobalValue::GUID)));
380 Hasher.update(ArrayRef((const uint8_t *)&Entry.second,
381 sizeof(GlobalValue::LinkageTypes)));
382 }
383
384 // Include the hash for the preserved symbols.
385 for (auto &Entry : PreservedSymbols) {
386 if (DefinedGVSummaries.count(Entry))
387 Hasher.update(
388 ArrayRef((const uint8_t *)&Entry, sizeof(GlobalValue::GUID)));
389 }
390
391 for (auto &Entry : DefinedGVSummaries)
392 if (auto *GVS = dyn_cast(Entry.second))
393 AddUnsigned(GVS->isReadOnly());
314 llvm::lto::Config Conf;
315 Conf.OptLevel = OptLevel;
316 Conf.Options = TMBuilder.Options;
317 Conf.CPU = TMBuilder.MCpu;
318 Conf.MAttrs.push_back(TMBuilder.MAttr);
319 Conf.RelocModel = TMBuilder.RelocModel;
320 Conf.CGOptLevel = TMBuilder.CGOptLevel;
321 Conf.Freestanding = Freestanding;
322 SmallString<40> Key;
323 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
324 ResolvedODR, DefinedGVSummaries);
394325
395326 // This choice of file name allows the cache to be pruned (see pruneCache()
396327 // in include/llvm/Support/CachePruning.h).
397 sys::path::append(EntryPath, CachePath,
398 "llvmcache-" + toHex(Hasher.result()));
328 sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
399329 }
400330
401331 // Access the path to this entry in the cache.
997927 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
998928 ImportLists[ModuleIdentifier], ExportList,
999929 ResolvedODR[ModuleIdentifier],
1000 DefinedGVSummaries, GUIDPreservedSymbols,
1001 OptLevel, Freestanding, TMBuilder);
930 DefinedGVSummaries, OptLevel, Freestanding,
931 TMBuilder);
1002932 auto CacheEntryPath = CacheEntry.getEntryPath();
1003933
1004934 {