llvm.org GIT mirror llvm / 23f8193
[ThinLTO/gold] Write empty imports even for modules with symbols Summary: ThinLTO may skip object for other reasons, e.g. if there is no summary. Reviewers: pcc, eugenis Subscribers: mehdi_amini, inglorion, eraman, llvm-commits Differential Revision: https://reviews.llvm.org/D42514 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@323818 91177308-0d34-0410-b5e6-96231b3b80d8 Vitaly Buka 2 years ago
5 changed file(s) with 59 addition(s) and 39 deletion(s). Raw diff Collapse all Expand all
209209 /// appends ".thinlto.bc" and writes the index to that path. If
210210 /// ShouldEmitImportsFiles is true it also writes a list of imported files to a
211211 /// similar path with ".imports" appended instead.
212 /// OnWrite is callback which receives module identifier and notifies LTO user
213 /// that index file for the module (and optionally imports file) was created.
214 using IndexWriteCallback = std::function;
212215 ThinBackend createWriteIndexesThinBackend(std::string OldPrefix,
213216 std::string NewPrefix,
214217 bool ShouldEmitImportsFiles,
215 std::string LinkedObjectsFile);
218 std::string LinkedObjectsFile,
219 IndexWriteCallback OnWrite);
216220
217221 /// This class implements a resolution-based interface to LLVM's LTO
218222 /// functionality. It supports regular LTO, parallel LTO code generation and
10321032 std::string LinkedObjectsFileName;
10331033 std::unique_ptr LinkedObjectsFile;
10341034
1035 lto::IndexWriteCallback OnWrite;
1036
10351037 public:
10361038 WriteIndexesThinBackend(
10371039 Config &Conf, ModuleSummaryIndex &CombinedIndex,
10381040 const StringMap &ModuleToDefinedGVSummaries,
10391041 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1040 std::string LinkedObjectsFileName)
1042 std::string LinkedObjectsFileName, lto::IndexWriteCallback OnWrite)
10411043 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
10421044 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
10431045 ShouldEmitImportsFiles(ShouldEmitImportsFiles),
1044 LinkedObjectsFileName(LinkedObjectsFileName) {}
1046 LinkedObjectsFileName(LinkedObjectsFileName), OnWrite(OnWrite) {}
10451047
10461048 Error start(
10471049 unsigned Task, BitcodeModule BM,
10741076 return errorCodeToError(EC);
10751077 WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
10761078
1077 if (ShouldEmitImportsFiles)
1078 return errorCodeToError(
1079 EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList));
1079 if (ShouldEmitImportsFiles) {
1080 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList);
1081 if (EC)
1082 return errorCodeToError(EC);
1083 }
1084
1085 if (OnWrite)
1086 OnWrite(ModulePath);
10801087 return Error::success();
10811088 }
10821089
10871094 ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix,
10881095 std::string NewPrefix,
10891096 bool ShouldEmitImportsFiles,
1090 std::string LinkedObjectsFile) {
1097 std::string LinkedObjectsFile,
1098 IndexWriteCallback OnWrite) {
10911099 return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
10921100 const StringMap &ModuleToDefinedGVSummaries,
10931101 AddStreamFn AddStream, NativeObjectCache Cache) {
10941102 return llvm::make_unique(
10951103 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
1096 ShouldEmitImportsFiles, LinkedObjectsFile);
1104 ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
10971105 };
10981106 }
10991107
0 ; RUN: rm -f %t*.o.thinlto.bc
1 ; RUN: rm -f %t*.o.imports
2
3 ; First generate bitcode with a module summary index for each file
1 ; RUN: opt -module-summary %s -o %t.o
4 ; RUN: opt -module-summary %s -o %t1.o
25 ; RUN: opt -module-summary %p/Inputs/thinlto_emit_linked_objects.ll -o %t2.o
6 ; RUN: opt %s -o %t3.o
37
48 ; Next do the ThinLink step, specifying thinlto-index-only so that the gold
59 ; plugin exits after generating individual indexes. The objects the linker
610 ; decided to include in the link should be emitted into the file specified
7 ; after 'thinlto-index-only='. In this version of the test, only %t.o will
11 ; after 'thinlto-index-only='. In this version of the test, only %t1.o will
812 ; be included in the link, and not %t2.o since it is within
913 ; a library (--start-lib/--end-lib pair) and not strongly referenced.
1014 ; Note that the support for detecting this is in gold v1.12.
11 ; RUN: rm -f %t.o.thinlto.bc
12 ; RUN: rm -f %t2.o.thinlto.bc
13 ; RUN: rm -f %t.o.imports
14 ; RUN: rm -f %t2.o.imports
1515 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
1616 ; RUN: --plugin-opt=thinlto \
17 ; RUN: --plugin-opt=thinlto-index-only=%t3 \
17 ; RUN: --plugin-opt=thinlto-index-only=%t.index \
1818 ; RUN: --plugin-opt=thinlto-emit-imports-files \
1919 ; RUN: -m elf_x86_64 \
2020 ; RUN: -o %t4 \
21 ; RUN: %t.o \
21 ; RUN: %t1.o %t3.o \
2222 ; RUN: --start-lib %t2.o --end-lib
2323
2424 ; Ensure that the expected output files are created, even for the file
2525 ; the linker decided not to include in the link.
26 ; RUN: ls %t.o.thinlto.bc
26 ; RUN: ls %t1.o.thinlto.bc
2727 ; RUN: ls %t2.o.thinlto.bc
28 ; RUN: ls %t.o.imports
28 ; RUN: ls %t3.o.thinlto.bc
29 ; RUN: ls %t1.o.imports
2930 ; RUN: ls %t2.o.imports
31 ; RUN: ls %t3.o.imports
3032
31 ; RUN: cat %t3 | FileCheck %s
32 ; CHECK: thinlto_emit_linked_objects.ll.tmp.o
33 ; RUN: cat %t.index | FileCheck %s
34 ; CHECK: thinlto_emit_linked_objects.ll.tmp1.o
3335 ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o
36 ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp3.o
3437
3538 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
3639 target triple = "x86_64-unknown-linux-gnu"
749749 std::tie(OldPrefix, NewPrefix) = PrefixReplace.split(';');
750750 }
751751
752 static std::unique_ptr createLTO() {
752 static std::unique_ptr createLTO(IndexWriteCallback OnIndexWrite) {
753753 Config Conf;
754754 ThinBackend Backend;
755755
776776 getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
777777 Backend = createWriteIndexesThinBackend(
778778 OldPrefix, NewPrefix, options::thinlto_emit_imports_files,
779 options::thinlto_linked_objects_file);
779 options::thinlto_linked_objects_file, OnIndexWrite);
780780 }
781781
782782 Conf.OverrideTriple = options::triple;
825825 // final link. Frequently the distributed build system will want to
826826 // confirm that all expected outputs are created based on all of the
827827 // modules provided to the linker.
828 static void writeEmptyDistributedBuildOutputs(std::string &ModulePath,
829 std::string &OldPrefix,
830 std::string &NewPrefix) {
828 static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
829 const std::string &OldPrefix,
830 const std::string &NewPrefix) {
831831 std::string NewModulePath =
832832 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
833833 std::error_code EC;
864864 // through Lto->run().
865865 DenseMap> HandleToInputFile;
866866
867 std::unique_ptr Lto = createLTO();
867 // Owns string objects and tells if index file was already created.
868 StringMap ObjectToIndexFileState;
869
870 std::unique_ptr Lto =
871 createLTO([&ObjectToIndexFileState](const std::string &Identifier) {
872 ObjectToIndexFileState[Identifier] = true;
873 });
868874
869875 std::string OldPrefix, NewPrefix;
870876 if (options::thinlto_index_only)
872878
873879 std::string OldSuffix, NewSuffix;
874880 getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix);
875 // Set for owning string objects used as buffer identifiers.
876 StringSet<> ObjectFilenames;
877881
878882 for (claimed_file &F : Modules) {
879883 if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
880884 HandleToInputFile.insert(std::make_pair(
881885 F.leader_handle, llvm::make_unique(F.handle)));
882 const void *View = getSymbolsAndView(F);
883886 // In case we are thin linking with a minimized bitcode file, ensure
884887 // the module paths encoded in the index reflect where the backends
885888 // will locate the full bitcode files for compiling/importing.
886889 std::string Identifier =
887890 getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix);
888 auto ObjFilename = ObjectFilenames.insert(Identifier);
891 auto ObjFilename = ObjectToIndexFileState.insert({Identifier, false});
889892 assert(ObjFilename.second);
890 if (!View) {
891 if (options::thinlto_index_only)
892 // Write empty output files that may be expected by the distributed
893 // build system.
894 writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix);
895 continue;
896 }
897 addModule(*Lto, F, View, ObjFilename.first->first());
893 if (const void *View = getSymbolsAndView(F))
894 addModule(*Lto, F, View, ObjFilename.first->first());
898895 }
899896
900897 SmallString<128> Filename;
930927 Cache = check(localCache(options::cache_dir, AddBuffer));
931928
932929 check(Lto->run(AddStream, Cache));
930
931 // Write empty output files that may be expected by the distributed build
932 // system.
933 if (options::thinlto_index_only)
934 for (auto &Identifier : ObjectToIndexFileState)
935 if (!Identifier.getValue())
936 writeEmptyDistributedBuildOutputs(Identifier.getKey(), OldPrefix,
937 NewPrefix);
933938
934939 if (options::TheOutputType == options::OT_DISABLE ||
935940 options::TheOutputType == options::OT_BC_ONLY)
242242
243243 ThinBackend Backend;
244244 if (ThinLTODistributedIndexes)
245 Backend = createWriteIndexesThinBackend("", "", true, "");
245 Backend = createWriteIndexesThinBackend("", "", true, "", {});
246246 else
247247 Backend = createInProcessThinBackend(Threads);
248248 LTO Lto(std::move(Conf), std::move(Backend));