llvm.org GIT mirror llvm / b81a1e9
[ThinLTO] Emit files for distributed builds for all modules With the new LTO API in r278338, we stopped emitting the individual index files and imports files for some modules in the distributed backend case (thinlto-index-only plugin option). Specifically, this is when the linker decides not to include a module in the link, because it was in an archive library and did not have a strong reference to it. Not creating the expected output files makes the distributed build system implementation more difficult, in terms of checking for the expected outputs of the thin link, and scheduling the backend jobs. To address this, the gold-plugin will write dummy empty .thinlto.bc and .imports files for modules not included in the link (which LTO never sees). Augmented a gold v1.12+ test, since that version of gold has the handling for notifying on modules not being included in the link. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@282100 91177308-0d34-0410-b5e6-96231b3b80d8 Teresa Johnson 4 years ago
4 changed file(s) with 77 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
8383 function_ref isExported);
8484
8585 namespace lto {
86
87 /// Given the original \p Path to an output file, replace any path
88 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
89 /// resulting directory if it does not yet exist.
90 std::string getThinLTOOutputFile(const std::string &Path,
91 const std::string &OldPrefix,
92 const std::string &NewPrefix);
8693
8794 class LTO;
8895 struct SymbolResolution;
608608 };
609609 }
610610
611 // Given the original \p Path to an output file, replace any path
612 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
613 // resulting directory if it does not yet exist.
614 std::string lto::getThinLTOOutputFile(const std::string &Path,
615 const std::string &OldPrefix,
616 const std::string &NewPrefix) {
617 if (OldPrefix.empty() && NewPrefix.empty())
618 return Path;
619 SmallString<128> NewPath(Path);
620 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
621 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
622 if (!ParentPath.empty()) {
623 // Make sure the new directory exists, creating it if necessary.
624 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
625 llvm::errs() << "warning: could not create directory '" << ParentPath
626 << "': " << EC.message() << '\n';
627 }
628 return NewPath.str();
629 }
630
611631 class WriteIndexesThinBackend : public ThinBackendProc {
612632 std::string OldPrefix, NewPrefix;
613633 bool ShouldEmitImportsFiles;
625645 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
626646 ShouldEmitImportsFiles(ShouldEmitImportsFiles),
627647 LinkedObjectsFileName(LinkedObjectsFileName) {}
628
629 /// Given the original \p Path to an output file, replace any path
630 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
631 /// resulting directory if it does not yet exist.
632 std::string getThinLTOOutputFile(const std::string &Path,
633 const std::string &OldPrefix,
634 const std::string &NewPrefix) {
635 if (OldPrefix.empty() && NewPrefix.empty())
636 return Path;
637 SmallString<128> NewPath(Path);
638 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
639 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
640 if (!ParentPath.empty()) {
641 // Make sure the new directory exists, creating it if necessary.
642 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
643 llvm::errs() << "warning: could not create directory '" << ParentPath
644 << "': " << EC.message() << '\n';
645 }
646 return NewPath.str();
647 }
648648
649649 Error start(
650650 unsigned Task, MemoryBufferRef MBRef,
88 ; be included in the link, and not %t2.o since it is within
99 ; a library (--start-lib/--end-lib pair) and not strongly referenced.
1010 ; Note that the support for detecting this is in gold v1.12.
11 ; RUN: rm -f %t.o.thinlto.bc
12 ; RUN: rm -f %t2.o.thinlto.bc
13 ; RUN: rm -f %t.o.imports
14 ; RUN: rm -f %t2.o.imports
1115 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
1216 ; RUN: --plugin-opt=thinlto \
1317 ; RUN: --plugin-opt=thinlto-index-only=%t3 \
18 ; RUN: --plugin-opt=thinlto-emit-imports-files \
1419 ; RUN: -m elf_x86_64 \
1520 ; RUN: -o %t4 \
1621 ; RUN: %t.o \
1722 ; RUN: --start-lib %t2.o --end-lib
23
24 ; Ensure that the expected output files are created, even for the file
25 ; the linker decided not to include in the link.
26 ; RUN: ls %t.o.thinlto.bc
27 ; RUN: ls %t2.o.thinlto.bc
28 ; RUN: ls %t.o.imports
29 ; RUN: ls %t2.o.imports
1830
1931 ; RUN: cat %t3 | FileCheck %s
2032 ; CHECK: thinlto_emit_linked_objects.ll.tmp.o
751751 ParallelCodeGenParallelismLevel);
752752 }
753753
754 // Write empty files that may be expected by a distributed build
755 // system when invoked with thinlto_index_only. This is invoked when
756 // the linker has decided not to include the given module in the
757 // final link. Frequently the distributed build system will want to
758 // confirm that all expected outputs are created based on all of the
759 // modules provided to the linker.
760 static void writeEmptyDistributedBuildOutputs(std::string &ModulePath,
761 std::string &OldPrefix,
762 std::string &NewPrefix) {
763 std::string NewModulePath =
764 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
765 std::error_code EC;
766 {
767 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
768 sys::fs::OpenFlags::F_None);
769 if (EC)
770 message(LDPL_FATAL, "Failed to write '%s': %s",
771 (NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
772 }
773 if (options::thinlto_emit_imports_files) {
774 raw_fd_ostream OS(NewModulePath + ".imports", EC,
775 sys::fs::OpenFlags::F_None);
776 if (EC)
777 message(LDPL_FATAL, "Failed to write '%s': %s",
778 (NewModulePath + ".imports").c_str(), EC.message().c_str());
779 }
780 }
781
754782 /// gold informs us that all symbols have been read. At this point, we use
755783 /// get_symbols to see if any of our definitions have been overridden by a
756784 /// native object file. Then, perform optimization and codegen.
770798
771799 std::unique_ptr Lto = createLTO();
772800
801 std::string OldPrefix, NewPrefix;
802 if (options::thinlto_index_only)
803 getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
804
773805 for (claimed_file &F : Modules) {
774806 if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
775807 HandleToInputFile.insert(std::make_pair(
776808 F.leader_handle, llvm::make_unique(F.handle)));
777809 const void *View = getSymbolsAndView(F);
778 if (!View)
810 if (!View) {
811 if (options::thinlto_index_only)
812 // Write empty output files that may be expected by the distributed
813 // build system.
814 writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix);
779815 continue;
816 }
780817 addModule(*Lto, F, View);
781818 }
782819