llvm.org GIT mirror llvm / 08d0e94
[ThinLTO] Add support for emitting minimized bitcode for thin link Summary: The cumulative size of the bitcode files for a very large application can be huge, particularly with -g. In a distributed build environment, all of these files must be sent to the remote build node that performs the thin link step, and this can exceed size limits. The thin link actually only needs the summary along with a bitcode symbol table. Until we have a proper bitcode symbol table, simply stripping the debug metadata results in significant size reduction. Add support for an option to additionally emit minimized bitcode modules, just for use in the thin link step, which for now just strips all debug metadata. I plan to add a cc1 option so this can be invoked easily during the compile step. However, care must be taken to ensure that these minimized thin link bitcode files produce the same index as with the original bitcode files, as these original bitcode files will be used in the backends. Specifically: 1) The module hash used for caching is typically produced by hashing the written bitcode, and we want to include the hash that would correspond to the original bitcode file. This is because we want to ensure that changes in the stripped portions affect caching. Added plumbing to emit the same module hash in the minimized thin link bitcode file. 2) The module paths in the index are constructed from the module ID of each thin linked bitcode, and typically is automatically generated from the input file path. This is the path used for finding the modules to import from, and obviously we need this to point to the original bitcode files. Added gold-plugin support to take a suffix replacement during the thin link that is used to override the identifier on the MemoryBufferRef constructed from the loaded thin link bitcode file. The assumption is that the build system can specify that the minimized bitcode file has a name that is similar but uses a different suffix (e.g. out.thinlink.bc instead of out.o). Added various tests to ensure that we get identical index files out of the thin link step. Reviewers: mehdi_amini, pcc Subscribers: Prazek, llvm-commits Differential Revision: https://reviews.llvm.org/D31027 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@298638 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson 3 years ago
13 changed file(s) with 348 addition(s) and 66 deletion(s). Raw diff Collapse all Expand all
4242 ///
4343 /// \p GenerateHash enables hashing the Module and including the hash in the
4444 /// bitcode (currently for use in ThinLTO incremental build).
45 ///
46 /// If \p ModHash is non-null, when GenerateHash is true, the resulting
47 /// hash is written into ModHash. When GenerateHash is false, that value
48 /// is used as the hash instead of computing from the generated bitcode.
49 /// Can be used to produce the same module hash for a minimized bitcode
50 /// used just for the thin link as in the regular full bitcode that will
51 /// be used in the backend.
4552 void writeModule(const Module *M, bool ShouldPreserveUseListOrder = false,
4653 const ModuleSummaryIndex *Index = nullptr,
47 bool GenerateHash = false);
54 bool GenerateHash = false, ModuleHash *ModHash = nullptr);
4855 };
4956
5057 /// \brief Write the specified module to the specified raw output stream.
6168 ///
6269 /// \p GenerateHash enables hashing the Module and including the hash in the
6370 /// bitcode (currently for use in ThinLTO incremental build).
71 ///
72 /// If \p ModHash is non-null, when GenerateHash is true, the resulting
73 /// hash is written into ModHash. When GenerateHash is false, that value
74 /// is used as the hash instead of computing from the generated bitcode.
75 /// Can be used to produce the same module hash for a minimized bitcode
76 /// used just for the thin link as in the regular full bitcode that will
77 /// be used in the backend.
6478 void WriteBitcodeToFile(const Module *M, raw_ostream &Out,
6579 bool ShouldPreserveUseListOrder = false,
6680 const ModuleSummaryIndex *Index = nullptr,
67 bool GenerateHash = false);
81 bool GenerateHash = false,
82 ModuleHash *ModHash = nullptr);
6883
6984 /// Write the specified module summary index to the given raw output stream,
7085 /// where it will be written in a new bitcode block. This is used when
8787 }
8888
8989 /// Parse the module summary index out of an IR file and return the module
90 /// summary index object if found, or nullptr if not.
90 /// summary index object if found, or nullptr if not. If Identifier is
91 /// non-empty, it is used as the module ID (module path) in the resulting
92 /// index. This can be used when the index is being read from a file
93 /// containing minimized bitcode just for the thin link.
9194 Expected>
92 getModuleSummaryIndexForFile(StringRef Path);
95 getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier = "");
9396 }
9497
9598 #endif
263263 ModulePass *createSampleProfileLoaderPass(StringRef Name);
264264
265265 /// Write ThinLTO-ready bitcode to Str.
266 ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str);
266 ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str,
267 raw_ostream *ThinLinkOS = nullptr);
267268
268269 } // End llvm namespace
269270
107107 /// True if a module hash record should be written.
108108 bool GenerateHash;
109109
110 /// If non-null, when GenerateHash is true, the resulting hash is written
111 /// into ModHash. When GenerateHash is false, that specified value
112 /// is used as the hash instead of computing from the generated bitcode.
113 /// Can be used to produce the same module hash for a minimized bitcode
114 /// used just for the thin link as in the regular full bitcode that will
115 /// be used in the backend.
116 ModuleHash *ModHash;
117
110118 /// The start bit of the identification block.
111119 uint64_t BitcodeStartBit;
112120
123131 /// writing to the provided \p Buffer.
124132 ModuleBitcodeWriter(const Module *M, SmallVectorImpl &Buffer,
125133 BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
126 const ModuleSummaryIndex *Index, bool GenerateHash)
134 const ModuleSummaryIndex *Index, bool GenerateHash,
135 ModuleHash *ModHash = nullptr)
127136 : BitcodeWriterBase(Stream), Buffer(Buffer), M(*M),
128137 VE(*M, ShouldPreserveUseListOrder), Index(Index),
129 GenerateHash(GenerateHash), BitcodeStartBit(Stream.GetCurrentBitNo()) {
138 GenerateHash(GenerateHash), ModHash(ModHash),
139 BitcodeStartBit(Stream.GetCurrentBitNo()) {
130140 // Assign ValueIds to any callee values in the index that came from
131141 // indirect call profiles and were recorded as a GUID not a Value*
132142 // (which would have been assigned an ID by the ValueEnumerator).
37773787 void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) {
37783788 // Emit the module's hash.
37793789 // MODULE_CODE_HASH: [5*i32]
3780 SHA1 Hasher;
3781 Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos],
3782 Buffer.size() - BlockStartPos));
3783 StringRef Hash = Hasher.result();
3784 uint32_t Vals[5];
3785 for (int Pos = 0; Pos < 20; Pos += 4) {
3786 Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos);
3787 }
3788
3789 // Emit the finished record.
3790 Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals);
3790 if (GenerateHash) {
3791 SHA1 Hasher;
3792 uint32_t Vals[5];
3793 Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos],
3794 Buffer.size() - BlockStartPos));
3795 StringRef Hash = Hasher.result();
3796 for (int Pos = 0; Pos < 20; Pos += 4) {
3797 Vals[Pos / 4] = support::endian::read32be(Hash.data() + Pos);
3798 }
3799
3800 // Emit the finished record.
3801 Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals);
3802
3803 if (ModHash)
3804 // Save the written hash value.
3805 std::copy(std::begin(Vals), std::end(Vals), std::begin(*ModHash));
3806 } else if (ModHash)
3807 Stream.EmitRecord(bitc::MODULE_CODE_HASH, ArrayRef(*ModHash));
37913808 }
37923809
37933810 void ModuleBitcodeWriter::write() {
38483865 writeValueSymbolTable(M.getValueSymbolTable(),
38493866 /* IsModuleLevel */ true, &FunctionToBitcodeIndex);
38503867
3851 if (GenerateHash) {
3852 writeModuleHash(BlockStartPos);
3853 }
3868 writeModuleHash(BlockStartPos);
38543869
38553870 Stream.ExitBlock();
38563871 }
39413956 void BitcodeWriter::writeModule(const Module *M,
39423957 bool ShouldPreserveUseListOrder,
39433958 const ModuleSummaryIndex *Index,
3944 bool GenerateHash) {
3945 ModuleBitcodeWriter ModuleWriter(
3946 M, Buffer, *Stream, ShouldPreserveUseListOrder, Index, GenerateHash);
3959 bool GenerateHash, ModuleHash *ModHash) {
3960 ModuleBitcodeWriter ModuleWriter(M, Buffer, *Stream,
3961 ShouldPreserveUseListOrder, Index,
3962 GenerateHash, ModHash);
39473963 ModuleWriter.write();
39483964 }
39493965
39523968 void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
39533969 bool ShouldPreserveUseListOrder,
39543970 const ModuleSummaryIndex *Index,
3955 bool GenerateHash) {
3971 bool GenerateHash, ModuleHash *ModHash) {
39563972 SmallVector Buffer;
39573973 Buffer.reserve(256*1024);
39583974
39633979 Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0);
39643980
39653981 BitcodeWriter Writer(Buffer);
3966 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash);
3982 Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
3983 ModHash);
39673984
39683985 if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
39693986 emitDarwinBCHeaderAndTrailer(Buffer, TT);
9595 // Parse the module summary index out of an IR file and return the summary
9696 // index object if found, or nullptr if not.
9797 Expected>
98 llvm::getModuleSummaryIndexForFile(StringRef Path) {
98 llvm::getModuleSummaryIndexForFile(StringRef Path, StringRef Identifier) {
9999 ErrorOr> FileOrErr =
100100 MemoryBuffer::getFileOrSTDIN(Path);
101101 std::error_code EC = FileOrErr.getError();
102102 if (EC)
103103 return errorCodeToError(EC);
104 MemoryBufferRef BufferRef = (FileOrErr.get())->getMemBufferRef();
104 std::unique_ptr MemBuffer = std::move(FileOrErr.get());
105 // If Identifier is non-empty, use it as the buffer identifier, which
106 // will become the module path in the index.
107 if (Identifier.empty())
108 Identifier = MemBuffer->getBufferIdentifier();
109 MemoryBufferRef BufferRef(MemBuffer->getBuffer(), Identifier);
105110 if (IgnoreEmptyThinLTOIndexFile && !BufferRef.getBufferSize())
106111 return nullptr;
107112 Expected> ObjOrErr =
1313 //
1414 //===----------------------------------------------------------------------===//
1515
16 #include "llvm/Transforms/IPO.h"
1716 #include "llvm/Analysis/BasicAliasAnalysis.h"
1817 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
1918 #include "llvm/Analysis/TypeMetadataUtils.h"
2423 #include "llvm/IR/Module.h"
2524 #include "llvm/IR/PassManager.h"
2625 #include "llvm/Pass.h"
26 #include "llvm/Support/FileSystem.h"
2727 #include "llvm/Support/ScopedPrinter.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include "llvm/Transforms/IPO.h"
2830 #include "llvm/Transforms/IPO/FunctionAttrs.h"
2931 #include "llvm/Transforms/Utils/Cloning.h"
3032 using namespace llvm;
250252 // a multi-module bitcode file with the two parts to OS. Otherwise, write only a
251253 // regular LTO bitcode file to OS.
252254 void splitAndWriteThinLTOBitcode(
253 raw_ostream &OS, function_ref AARGetter,
254 Module &M) {
255 raw_ostream &OS, raw_ostream *ThinLinkOS,
256 function_ref AARGetter, Module &M) {
255257 std::string ModuleId = getModuleId(&M);
256258 if (ModuleId.empty()) {
257259 // We couldn't generate a module ID for this module, just write it out as a
258260 // regular LTO module.
259261 WriteBitcodeToFile(&M, OS);
262 if (ThinLinkOS)
263 // We don't have a ThinLTO part, but still write the module to the
264 // ThinLinkOS if requested so that the expected output file is produced.
265 WriteBitcodeToFile(&M, *ThinLinkOS);
260266 return;
261267 }
262268
333339
334340 simplifyExternals(*MergedM);
335341
336 SmallVector Buffer;
337 BitcodeWriter W(Buffer);
338342
339343 // FIXME: Try to re-use BSI and PFI from the original module here.
340344 ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, nullptr);
345
346 SmallVector Buffer;
347
348 BitcodeWriter W(Buffer);
349 // Save the module hash produced for the full bitcode, which will
350 // be used in the backends, and use that in the minimized bitcode
351 // produced for the full link.
352 ModuleHash ModHash = {{0}};
341353 W.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index,
342 /*GenerateHash=*/true);
343
354 /*GenerateHash=*/true, &ModHash);
344355 W.writeModule(MergedM.get());
345
346356 OS << Buffer;
357
358 // If a minimized bitcode module was requested for the thin link,
359 // strip the debug info (the merged module was already stripped above)
360 // and write it to the given OS.
361 if (ThinLinkOS) {
362 Buffer.clear();
363 BitcodeWriter W2(Buffer);
364 StripDebugInfo(M);
365 W2.writeModule(&M, /*ShouldPreserveUseListOrder=*/false, &Index,
366 /*GenerateHash=*/false, &ModHash);
367 W2.writeModule(MergedM.get());
368 *ThinLinkOS << Buffer;
369 }
347370 }
348371
349372 // Returns whether this module needs to be split because it uses type metadata.
358381 return false;
359382 }
360383
361 void writeThinLTOBitcode(raw_ostream &OS,
384 void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
362385 function_ref AARGetter,
363386 Module &M, const ModuleSummaryIndex *Index) {
364387 // See if this module has any type metadata. If so, we need to split it.
365388 if (requiresSplit(M))
366 return splitAndWriteThinLTOBitcode(OS, AARGetter, M);
389 return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
367390
368391 // Otherwise we can just write it out as a regular module.
392
393 // Save the module hash produced for the full bitcode, which will
394 // be used in the backends, and use that in the minimized bitcode
395 // produced for the full link.
396 ModuleHash ModHash = {{0}};
369397 WriteBitcodeToFile(&M, OS, /*ShouldPreserveUseListOrder=*/false, Index,
370 /*GenerateHash=*/true);
398 /*GenerateHash=*/true, &ModHash);
399 // If a minimized bitcode module was requested for the thin link,
400 // strip the debug info and write it to the given OS.
401 if (ThinLinkOS) {
402 StripDebugInfo(M);
403 WriteBitcodeToFile(&M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false,
404 Index,
405 /*GenerateHash=*/false, &ModHash);
406 }
371407 }
372408
373409 class WriteThinLTOBitcode : public ModulePass {
374410 raw_ostream &OS; // raw_ostream to print on
411 // The output stream on which to emit a minimized module for use
412 // just in the thin link, if requested.
413 raw_ostream *ThinLinkOS;
375414
376415 public:
377416 static char ID; // Pass identification, replacement for typeid
378 WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()) {
417 WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()), ThinLinkOS(nullptr) {
379418 initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
380419 }
381420
382 explicit WriteThinLTOBitcode(raw_ostream &o)
383 : ModulePass(ID), OS(o) {
421 explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS)
422 : ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) {
384423 initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
385424 }
386425
389428 bool runOnModule(Module &M) override {
390429 const ModuleSummaryIndex *Index =
391430 &(getAnalysis().getIndex());
392 writeThinLTOBitcode(OS, LegacyAARGetter(*this), M, Index);
431 writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index);
393432 return true;
394433 }
395434 void getAnalysisUsage(AnalysisUsage &AU) const override {
410449 INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode",
411450 "Write ThinLTO Bitcode", false, true)
412451
413 ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str) {
414 return new WriteThinLTOBitcode(Str);
415 }
452 ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str,
453 raw_ostream *ThinLinkOS) {
454 return new WriteThinLTOBitcode(Str, ThinLinkOS);
455 }
None ; RUN: opt -module-summary %s -o %t1.bc
1 ; RUN: opt -module-summary %p/Inputs/distributed_import.ll -o %t2.bc
0 ; Test distributed build thin link output from llvm-lto2
21
2 ; Generate bitcode files with summary, as well as minimized bitcode without
3 ; the debug metadata for the thin link.
4 ; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
5 ; RUN: opt -thinlto-bc %p/Inputs/distributed_import.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
6
7 ; First perform the thin link on the normal bitcode file.
38 ; RUN: llvm-lto2 %t1.bc %t2.bc -o %t.o -save-temps \
49 ; RUN: -thinlto-distributed-indexes \
510 ; RUN: -r=%t1.bc,g, \
611 ; RUN: -r=%t1.bc,f,px \
712 ; RUN: -r=%t2.bc,g,px
8 ; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out
13 ; RUN: opt -function-import -summary-file %t1.bc.thinlto.bc %t1.bc -o %t1.out
914 ; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out
1015 ; RUN: llvm-dis -o - %t2.out | FileCheck %s
11 ; CHECK: @G.llvm.0
16
17 ; Save the generated index files.
18 ; RUN: cp %t1.bc.thinlto.bc %t1.bc.thinlto.bc.orig
19 ; RUN: cp %t2.bc.thinlto.bc %t2.bc.thinlto.bc.orig
20
21 ; Copy the minimized bitcode to the regular bitcode path so the module
22 ; paths in the index are the same (save the regular bitcode for use again
23 ; further down).
24 ; RUN: cp %t1.bc %t1.bc.sv
25 ; RUN: cp %t1.thinlink.bc %t1.bc
26 ; RUN: cp %t2.bc %t2.bc.sv
27 ; RUN: cp %t2.thinlink.bc %t2.bc
28
29 ; Next perform the thin link on the minimized bitcode files, and compare dumps
30 ; of the resulting indexes to the above dumps to ensure they are identical.
31 ; RUN: rm -f %t1.bc.thinlto.bc %t2.bc.thinlto.bc
32 ; RUN: llvm-lto2 %t1.bc %t2.bc -o %t.o -save-temps \
33 ; RUN: -thinlto-distributed-indexes \
34 ; RUN: -r=%t1.bc,g, \
35 ; RUN: -r=%t1.bc,f,px \
36 ; RUN: -r=%t2.bc,g,px
37 ; RUN: diff %t1.bc.thinlto.bc.orig %t1.bc.thinlto.bc
38 ; RUN: diff %t2.bc.thinlto.bc.orig %t2.bc.thinlto.bc
39
40 ; Make sure importing occurs as expected
41 ; RUN: cp %t1.bc.sv %t1.bc
42 ; RUN: cp %t2.bc.sv %t2.bc
43 ; RUN: opt -function-import -summary-file %t2.bc.thinlto.bc %t2.bc -o %t2.out
44 ; RUN: llvm-dis -o - %t2.out | FileCheck %s
45
46 ; CHECK: @G.llvm.
1247
1348 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
1449
1954 call i32 (...) @g()
2055 ret void
2156 }
57
58 !llvm.dbg.cu = !{}
59
60 !1 = !{i32 2, !"Debug Info Version", i32 3}
61 !llvm.module.flags = !{!1}
None ; RUN: opt -thinlto-bc -o %t %s
1 ; RUN: llvm-dis -o - %t | FileCheck %s
2 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
0 ; Generate bitcode files with summary, as well as minimized bitcode without
1 ; the debug metadata for the thin link.
2 ; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t.thinlink.bc -o %t.bc %s
3 ; RUN: llvm-dis -o - %t.bc | FileCheck %s
4 ; RUN: llvm-dis -o - %t.thinlink.bc | FileCheck --check-prefix=NODEBUG %s
5 ; RUN: llvm-bcanalyzer -dump %t.bc | FileCheck --check-prefix=BCA %s
6
7 ; Make sure the combined index files produced by both the normal and the
8 ; thin link bitcode files are identical
9 ; RUN: llvm-lto -thinlto -o %t3 %t.bc
10 ; Copy the minimized bitcode to the regular bitcode path so the module
11 ; paths in the index are the same (save and restore the regular bitcode
12 ; for use again further down).
13 ; RUN: mv %t.bc %t.bc.sv
14 ; RUN: cp %t.thinlink.bc %t.bc
15 ; RUN: llvm-lto -thinlto -o %t4 %t.bc
16 ; RUN: mv %t.bc.sv %t.bc
17 ; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
18
19 ; Try again using -thinlto-action to produce combined index
20 ; RUN: rm -f %t3.thinlto.bc %t4.thinlto.bc
21 ; RUN: llvm-lto -thinlto-action=thinlink -o %t3.thinlto.bc %t.bc
22 ; Copy the minimized bitcode to the regular bitcode path so the module
23 ; paths in the index are the same.
24 ; RUN: cp %t.thinlink.bc %t.bc
25 ; RUN: llvm-lto -thinlto-action=thinlink -o %t4.thinlto.bc %t.bc
26 ; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
327
428 ; BCA:
529
1034 define void @f() {
1135 ret void
1236 }
37
38 ; CHECK: !llvm.dbg.cu
39 ; NODEBUG-NOT: !llvm.dbg.cu
40 !llvm.dbg.cu = !{}
41
42 !1 = !{i32 2, !"Debug Info Version", i32 3}
43 !llvm.module.flags = !{!1}
None ; RUN: opt -thinlto-bc -o %t %s
1 ; RUN: llvm-modextract -b -n 0 -o %t0 %t
2 ; RUN: llvm-modextract -b -n 1 -o %t1 %t
0 ; Generate bitcode files with summary, as well as minimized bitcode without
1 ; the debug metadata for the thin link.
2 ; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s
3 ; RUN: llvm-modextract -b -n 0 -o %t0.bc %t
4 ; RUN: llvm-modextract -b -n 1 -o %t1.bc %t
5 ; RUN: llvm-modextract -b -n 0 -o %t0.thinlink.bc %t2
6 ; RUN: llvm-modextract -b -n 1 -o %t1.thinlink.bc %t2
37 ; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
4 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s
5 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s
6 ; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s
7 ; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s
8 ; RUN: llvm-dis -o - %t0.bc | FileCheck --check-prefix=M0 %s
9 ; RUN: llvm-dis -o - %t1.bc | FileCheck --check-prefix=M1 %s
10 ; RUN: llvm-dis -o - %t0.thinlink.bc | FileCheck --check-prefix=NODEBUG %s
11 ; RUN: llvm-dis -o - %t1.thinlink.bc | FileCheck --check-prefix=NODEBUG %s
12 ; RUN: llvm-bcanalyzer -dump %t0.bc | FileCheck --check-prefix=BCA0 %s
13 ; RUN: llvm-bcanalyzer -dump %t1.bc | FileCheck --check-prefix=BCA1 %s
14
15 ; Make sure the combined index files produced by both the normal and the
16 ; thin link bitcode files are identical
17 ; RUN: llvm-lto -thinlto -o %t3 %t0.bc
18 ; Copy the minimized bitcode to the regular bitcode path so the module
19 ; paths in the index are the same.
20 ; RUN: cp %t0.thinlink.bc %t0.bc
21 ; RUN: llvm-lto -thinlto -o %t4 %t0.bc
22 ; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
823
924 ; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s)
1025
2843
2944 ; M0: !llvm.dbg.cu
3045 ; M1-NOT: !llvm.dbg.cu
46 ; NODEBUG-NOT: !llvm.dbg.cu
3147 !llvm.dbg.cu = !{}
3248
3349 !1 = !{i32 2, !"Debug Info Version", i32 3}
None ; RUN: opt -thinlto-bc -o %t %s
0 ; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s
11 ; RUN: llvm-dis -o - %t | FileCheck %s
22 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
3 ; When not splitting the module, the thin link bitcode file should simply be a
4 ; copy of the regular module.
5 ; RUN: diff %t %t2
36
47 ; BCA-NOT:
58
0 ; Test to make sure the thinlto-object-suffix-replace option is handled
1 ; correctly.
2
3 ; Generate bitcode file with summary, as well as a minimized bitcode without
4 ; the debug metadata for the thin link.
5 ; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.o
6
7 ; First perform the thin link on the normal bitcode file, and save the
8 ; resulting index.
9 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
10 ; RUN: -m elf_x86_64 \
11 ; RUN: --plugin-opt=thinlto \
12 ; RUN: --plugin-opt=thinlto-index-only \
13 ; RUN: -shared %t1.o -o %t3
14 ; RUN: cp %t1.o.thinlto.bc %t1.o.thinlto.bc.orig
15
16 ; Next perform the thin link on the minimized bitcode file, and compare dump
17 ; of the resulting index to the above dump to ensure they are identical.
18 ; RUN: rm -f %t1.o.thinlto.bc
19 ; Make sure it isn't inadvertently using the regular bitcode file.
20 ; RUN: rm -f %t1.o
21 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
22 ; RUN: -m elf_x86_64 \
23 ; RUN: --plugin-opt=thinlto \
24 ; RUN: --plugin-opt=thinlto-index-only \
25 ; RUN: --plugin-opt=thinlto-object-suffix-replace=".thinlink.bc;.o" \
26 ; RUN: -shared %t1.thinlink.bc -o %t3
27 ; RUN: diff %t1.o.thinlto.bc.orig %t1.o.thinlto.bc
28
29 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
30 target triple = "x86_64-unknown-linux-gnu"
31
32 define void @f() {
33 entry:
34 ret void
35 }
36
37 !llvm.dbg.cu = !{}
38
39 !1 = !{i32 2, !"Debug Info Version", i32 3}
40 !llvm.module.flags = !{!1}
163163 // corresponding bitcode file, will use a path formed by replacing the
164164 // bitcode file's path prefix matching oldprefix with newprefix.
165165 static std::string thinlto_prefix_replace;
166 // Option to control the name of modules encoded in the individual index
167 // files for a distributed backend. This enables the use of minimized
168 // bitcode files for the thin link, assuming the name of the full bitcode
169 // file used in the backend differs just in some part of the file suffix.
170 // If specified, expects a string of the form "oldsuffix:newsuffix".
171 static std::string thinlto_object_suffix_replace;
166172 // Optional path to a directory for caching ThinLTO objects.
167173 static std::string cache_dir;
168174 // Additional options to pass into the code generator.
205211 thinlto_prefix_replace = opt.substr(strlen("thinlto-prefix-replace="));
206212 if (thinlto_prefix_replace.find(';') == std::string::npos)
207213 message(LDPL_FATAL, "thinlto-prefix-replace expects 'old;new' format");
214 } else if (opt.startswith("thinlto-object-suffix-replace=")) {
215 thinlto_object_suffix_replace =
216 opt.substr(strlen("thinlto-object-suffix-replace="));
217 if (thinlto_object_suffix_replace.find(';') == std::string::npos)
218 message(LDPL_FATAL,
219 "thinlto-object-suffix-replace expects 'old;new' format");
208220 } else if (opt.startswith("cache-dir=")) {
209221 cache_dir = opt.substr(strlen("cache-dir="));
210222 } else if (opt.size() == 2 && opt[0] == 'O') {
565577 return View;
566578 }
567579
568 static void addModule(LTO &Lto, claimed_file &F, const void *View) {
569 MemoryBufferRef BufferRef(StringRef((const char *)View, F.filesize), F.name);
580 /// Parse the thinlto-object-suffix-replace option into the \p OldSuffix and
581 /// \p NewSuffix strings, if it was specified.
582 static void getThinLTOOldAndNewSuffix(std::string &OldSuffix,
583 std::string &NewSuffix) {
584 assert(options::thinlto_object_suffix_replace.empty() ||
585 options::thinlto_object_suffix_replace.find(";") != StringRef::npos);
586 StringRef SuffixReplace = options::thinlto_object_suffix_replace;
587 std::pair Split = SuffixReplace.split(";");
588 OldSuffix = Split.first.str();
589 NewSuffix = Split.second.str();
590 }
591
592 /// Given the original \p Path to an output file, replace any filename
593 /// suffix matching \p OldSuffix with \p NewSuffix.
594 static std::string getThinLTOObjectFileName(const std::string Path,
595 const std::string &OldSuffix,
596 const std::string &NewSuffix) {
597 if (OldSuffix.empty() && NewSuffix.empty())
598 return Path;
599 StringRef NewPath = Path;
600 NewPath.consume_back(OldSuffix);
601 std::string NewNewPath = NewPath.str() + NewSuffix;
602 return NewPath.str() + NewSuffix;
603 }
604
605 static void addModule(LTO &Lto, claimed_file &F, const void *View,
606 StringRef Filename) {
607 MemoryBufferRef BufferRef(StringRef((const char *)View, F.filesize),
608 Filename);
570609 Expected> ObjOrErr = InputFile::create(BufferRef);
571610
572611 if (!ObjOrErr)
788827 if (options::thinlto_index_only)
789828 getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
790829
830 std::string OldSuffix, NewSuffix;
831 getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix);
832 // Set for owning string objects used as buffer identifiers.
833 StringSet<> ObjectFilenames;
834
791835 for (claimed_file &F : Modules) {
792836 if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
793837 HandleToInputFile.insert(std::make_pair(
794838 F.leader_handle, llvm::make_unique(F.handle)));
795839 const void *View = getSymbolsAndView(F);
840 // In case we are thin linking with a minimized bitcode file, ensure
841 // the module paths encoded in the index reflect where the backends
842 // will locate the full bitcode files for compiling/importing.
843 std::string Identifier =
844 getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix);
845 auto ObjFilename = ObjectFilenames.insert(Identifier);
846 assert(ObjFilename.second);
796847 if (!View) {
797848 if (options::thinlto_index_only)
798849 // Write empty output files that may be expected by the distributed
799850 // build system.
800 writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix);
851 writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix);
801852 continue;
802853 }
803 addModule(*Lto, F, View);
854 addModule(*Lto, F, View, ObjFilename.first->first());
804855 }
805856
806857 SmallString<128> Filename;
101101 OutputThinLTOBC("thinlto-bc",
102102 cl::desc("Write output as ThinLTO-ready bitcode"));
103103
104 static cl::opt ThinLinkBitcodeFile(
105 "thin-link-bitcode-file", cl::value_desc("filename"),
106 cl::desc(
107 "A file in which to write minimized bitcode for the thin link only"));
108
104109 static cl::opt
105110 NoVerify("disable-verify", cl::desc("Do not run the verifier"), cl::Hidden);
106111
455460
456461 // Figure out what stream we are supposed to write to...
457462 std::unique_ptr Out;
463 std::unique_ptr ThinLinkOut;
458464 if (NoOutput) {
459465 if (!OutputFilename.empty())
460466 errs() << "WARNING: The -o (output filename) option is ignored when\n"
469475 if (EC) {
470476 errs() << EC.message() << '\n';
471477 return 1;
478 }
479
480 if (!ThinLinkBitcodeFile.empty()) {
481 ThinLinkOut.reset(
482 new tool_output_file(ThinLinkBitcodeFile, EC, sys::fs::F_None));
483 if (EC) {
484 errs() << EC.message() << '\n';
485 return 1;
486 }
472487 }
473488 }
474489
699714 report_fatal_error("Text output is incompatible with -module-hash");
700715 Passes.add(createPrintModulePass(*OS, "", PreserveAssemblyUseListOrder));
701716 } else if (OutputThinLTOBC)
702 Passes.add(createWriteThinLTOBitcodePass(*OS));
717 Passes.add(createWriteThinLTOBitcodePass(
718 *OS, ThinLinkOut ? &ThinLinkOut->os() : nullptr));
703719 else
704720 Passes.add(createBitcodeWriterPass(*OS, PreserveBitcodeUseListOrder,
705721 EmitSummaryIndex, EmitModuleHash));
746762 if (YamlFile)
747763 YamlFile->keep();
748764
765 if (ThinLinkOut)
766 ThinLinkOut->keep();
767
749768 return 0;
750769 }