llvm.org GIT mirror llvm / c1e784c
TransformUtils: Introduce module splitter. The module splitter splits a module into linkable partitions. It will be used to implement parallel LTO code generation. This initial version of the splitter does not attempt to deal with the somewhat subtle symbol visibility issues around module splitting. These will be dealt with in a future change. Differential Revision: http://reviews.llvm.org/D12132 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@245662 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 4 years ago
19 changed file(s) with 413 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
2222 #include "llvm/IR/ValueHandle.h"
2323 #include "llvm/IR/ValueMap.h"
2424 #include "llvm/Transforms/Utils/ValueMapper.h"
25 #include
2526
2627 namespace llvm {
2728
5152 Module *CloneModule(const Module *M);
5253 Module *CloneModule(const Module *M, ValueToValueMapTy &VMap);
5354
55 /// Return a copy of the specified module. The ShouldCloneDefinition function
56 /// controls whether a specific GlobalValue's definition is cloned. If the
57 /// function returns false, the module copy will contain an external reference
58 /// in place of the global definition.
59 Module *
60 CloneModule(const Module *M, ValueToValueMapTy &VMap,
61 std::function ShouldCloneDefinition);
62
5463 /// ClonedCodeInfo - This struct can be used to capture information about code
5564 /// being cloned, while it is being cloned.
5665 struct ClonedCodeInfo {
0 //===- SplitModule.h - Split a module into partitions -----------*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the function llvm::SplitModule, which splits a module
10 // into multiple linkable partitions. It can be used to implement parallel code
11 // generation for link-time optimization.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_TRANSFORMS_UTILS_SPLITMODULE_H
16 #define LLVM_TRANSFORMS_UTILS_SPLITMODULE_H
17
18 #include
19 #include
20
21 namespace llvm {
22
23 class Module;
24 class StringRef;
25
26 /// Splits the module M into N linkable partitions. The function ModuleCallback
27 /// is called N times passing each individual partition as the MPart argument.
28 ///
29 /// FIXME: This function does not deal with the somewhat subtle symbol
30 /// visibility issues around module splitting, including (but not limited to):
31 ///
32 /// - Internal symbols should not collide with symbols defined outside the
33 /// module.
34 /// - Internal symbols defined in module-level inline asm should be visible to
35 /// each partition.
36 void SplitModule(
37 std::unique_ptr M, unsigned N,
38 std::function MPart)> ModuleCallback);
39
40 } // End llvm namespace
41
42 #endif
3333 SimplifyIndVar.cpp
3434 SimplifyInstructions.cpp
3535 SimplifyLibCalls.cpp
36 SplitModule.cpp
3637 SymbolRewriter.cpp
3738 UnifyFunctionExitNodes.cpp
3839 Utils.cpp
3232 }
3333
3434 Module *llvm::CloneModule(const Module *M, ValueToValueMapTy &VMap) {
35 return CloneModule(M, VMap, [](const GlobalValue *GV) { return true; });
36 }
37
38 Module *llvm::CloneModule(
39 const Module *M, ValueToValueMapTy &VMap,
40 std::function ShouldCloneDefinition) {
3541 // First off, we need to create the new module.
3642 Module *New = new Module(M->getModuleIdentifier(), M->getContext());
3743 New->setDataLayout(M->getDataLayout());
6773 // Loop over the aliases in the module
6874 for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
6975 I != E; ++I) {
76 if (!ShouldCloneDefinition(I)) {
77 // An alias cannot act as an external reference, so we need to create
78 // either a function or a global variable depending on the value type.
79 // FIXME: Once pointee types are gone we can probably pick one or the
80 // other.
81 GlobalValue *GV;
82 if (I->getValueType()->isFunctionTy())
83 GV = Function::Create(cast(I->getValueType()),
84 GlobalValue::ExternalLinkage, I->getName(), New);
85 else
86 GV = new GlobalVariable(
87 *New, I->getValueType(), false, GlobalValue::ExternalLinkage,
88 (Constant *)nullptr, I->getName(), (GlobalVariable *)nullptr,
89 I->getThreadLocalMode(), I->getType()->getAddressSpace());
90 VMap[I] = GV;
91 // We do not copy attributes (mainly because copying between different
92 // kinds of globals is forbidden), but this is generally not required for
93 // correctness.
94 continue;
95 }
7096 auto *PTy = cast(I->getType());
7197 auto *GA = GlobalAlias::create(PTy, I->getLinkage(), I->getName(), New);
7298 GA->copyAttributesFrom(I);
80106 for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
81107 I != E; ++I) {
82108 GlobalVariable *GV = cast(VMap[I]);
109 if (!ShouldCloneDefinition(I)) {
110 // Skip after setting the correct linkage for an external reference.
111 GV->setLinkage(GlobalValue::ExternalLinkage);
112 continue;
113 }
83114 if (I->hasInitializer())
84115 GV->setInitializer(MapValue(I->getInitializer(), VMap));
85116 }
88119 //
89120 for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) {
90121 Function *F = cast(VMap[I]);
122 if (!ShouldCloneDefinition(I)) {
123 // Skip after setting the correct linkage for an external reference.
124 F->setLinkage(GlobalValue::ExternalLinkage);
125 continue;
126 }
91127 if (!I->isDeclaration()) {
92128 Function::arg_iterator DestI = F->arg_begin();
93129 for (Function::const_arg_iterator J = I->arg_begin(); J != I->arg_end();
108144 // And aliases
109145 for (Module::const_alias_iterator I = M->alias_begin(), E = M->alias_end();
110146 I != E; ++I) {
147 // We already dealt with undefined aliases above.
148 if (!ShouldCloneDefinition(I))
149 continue;
111150 GlobalAlias *GA = cast(VMap[I]);
112151 if (const Constant *C = I->getAliasee())
113152 GA->setAliasee(MapValue(C, VMap));
0 //===- SplitModule.cpp - Split a module into partitions -------------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the function llvm::SplitModule, which splits a module
10 // into multiple linkable partitions. It can be used to implement parallel code
11 // generation for link-time optimization.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Transforms/Utils/SplitModule.h"
16 #include "llvm/ADT/Hashing.h"
17 #include "llvm/IR/Function.h"
18 #include "llvm/IR/GlobalAlias.h"
19 #include "llvm/IR/GlobalObject.h"
20 #include "llvm/IR/GlobalValue.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/Support/MD5.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include "llvm/Transforms/Utils/Cloning.h"
25
26 using namespace llvm;
27
28 static void externalize(GlobalValue *GV) {
29 if (GV->hasLocalLinkage()) {
30 GV->setLinkage(GlobalValue::ExternalLinkage);
31 GV->setVisibility(GlobalValue::HiddenVisibility);
32 }
33
34 // Unnamed entities must be named consistently between modules. setName will
35 // give a distinct name to each such entity.
36 if (!GV->hasName())
37 GV->setName("__llvmsplit_unnamed");
38 }
39
40 // Returns whether GV should be in partition (0-based) I of N.
41 static bool isInPartition(const GlobalValue *GV, unsigned I, unsigned N) {
42 if (auto GA = dyn_cast(GV))
43 if (const GlobalObject *Base = GA->getBaseObject())
44 GV = Base;
45
46 StringRef Name;
47 if (const Comdat *C = GV->getComdat())
48 Name = C->getName();
49 else
50 Name = GV->getName();
51
52 // Partition by MD5 hash. We only need a few bits for evenness as the number
53 // of partitions will generally be in the 1-2 figure range; the low 16 bits
54 // are enough.
55 MD5 H;
56 MD5::MD5Result R;
57 H.update(Name);
58 H.final(R);
59 return (R[0] | (R[1] << 8)) % N == I;
60 }
61
62 void llvm::SplitModule(
63 std::unique_ptr M, unsigned N,
64 std::function MPart)> ModuleCallback) {
65 for (Function &F : *M)
66 externalize(&F);
67 for (GlobalVariable &GV : M->globals())
68 externalize(&GV);
69 for (GlobalAlias &GA : M->aliases())
70 externalize(&GA);
71
72 // FIXME: We should be able to reuse M as the last partition instead of
73 // cloning it.
74 for (unsigned I = 0; I != N; ++I) {
75 ValueToValueMapTy VMap;
76 std::unique_ptr MPart(
77 CloneModule(M.get(), VMap, [=](const GlobalValue *GV) {
78 return isInPartition(GV, I, N);
79 }));
80 if (I != 0)
81 MPart->setModuleInlineAsm("");
82 ModuleCallback(std::move(MPart));
83 }
84 }
4545 llvm-readobj
4646 llvm-rtdyld
4747 llvm-size
48 llvm-split
4849 llvm-symbolizer
4950 llvm-tblgen
5051 macho-dump
247247 r"\bllvm-readobj\b",
248248 r"\bllvm-rtdyld\b",
249249 r"\bllvm-size\b",
250 r"\bllvm-split\b",
250251 r"\bllvm-tblgen\b",
251252 r"\bllvm-c-test\b",
252253 r"\bmacho-dump\b",
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 ; CHECK0-DAG: @afoo = alias [2 x i8*]* @foo
5 ; CHECK1-DAG: @afoo = external global [2 x i8*]
6 @afoo = alias [2 x i8*]* @foo
7
8 ; CHECK0-DAG: declare void @abar()
9 ; CHECK1-DAG: @abar = alias void ()* @bar
10 @abar = alias void ()* @bar
11
12 @foo = global [2 x i8*] [i8* bitcast (void ()* @bar to i8*), i8* bitcast (void ()* @abar to i8*)]
13
14 define void @bar() {
15 store [2 x i8*] zeroinitializer, [2 x i8*]* @foo
16 store [2 x i8*] zeroinitializer, [2 x i8*]* @afoo
17 ret void
18 }
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 $foo = comdat any
5
6 ; CHECK0: define void @foo()
7 ; CHECK1: declare void @foo()
8 define void @foo() comdat {
9 call void @bar()
10 ret void
11 }
12
13 ; CHECK0: define void @bar()
14 ; CHECK1: declare void @bar()
15 define void @bar() comdat($foo) {
16 call void @foo()
17 ret void
18 }
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 ; CHECK0: define void @foo()
5 ; CHECK1: declare void @foo()
6 define void @foo() {
7 call void @bar()
8 ret void
9 }
10
11 ; CHECK0: declare void @bar()
12 ; CHECK1: define void @bar()
13 define void @bar() {
14 call void @foo()
15 ret void
16 }
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 ; CHECK0: @foo = global i8* bitcast
5 ; CHECK1: @foo = external global i8*
6 @foo = global i8* bitcast (i8** @bar to i8*)
7
8 ; CHECK0: @bar = external global i8*
9 ; CHECK1: @bar = global i8* bitcast
10 @bar = global i8* bitcast (i8** @foo to i8*)
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 ; CHECK0: define hidden void @foo()
5 ; CHECK1: declare hidden void @foo()
6 define internal void @foo() {
7 call void @bar()
8 ret void
9 }
10
11 ; CHECK0: declare void @bar()
12 ; CHECK1: define void @bar()
13 define void @bar() {
14 call void @foo()
15 ret void
16 }
0 ; RUN: llvm-split -o %t %s
1 ; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=CHECK0 %s
2 ; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=CHECK1 %s
3
4 ; CHECK0: declare hidden void @__llvmsplit_unnamed()
5 ; CHECK1: define hidden void @__llvmsplit_unnamed()
6 define internal void @0() {
7 ; CHECK1: call void @foo()
8 call void @foo()
9 ret void
10 }
11
12 ; CHECK0: declare hidden void @__llvmsplit_unnamed1()
13 ; CHECK1: define hidden void @__llvmsplit_unnamed1()
14 define internal void @1() {
15 ; CHECK1: call void @foo()
16 ; CHECK1: call void @foo()
17 call void @foo()
18 call void @foo()
19 ret void
20 }
21
22 ; CHECK0: define void @foo()
23 ; CHECK1: declare void @foo()
24 define void @foo() {
25 ; CHECK0: call void @__llvmsplit_unnamed1()
26 ; CHECK0: call void @__llvmsplit_unnamed()
27 call void @1()
28 call void @0()
29 ret void
30 }
3939 llvm-profdata
4040 llvm-rtdyld
4141 llvm-size
42 llvm-split
4243 macho-dump
4344 opt
4445 verify-uselistorder
3131 macho-dump llvm-objdump llvm-readobj llvm-rtdyld \
3232 llvm-dwarfdump llvm-cov llvm-size llvm-stress llvm-mcmarkup \
3333 llvm-profdata llvm-symbolizer obj2yaml yaml2obj llvm-c-test \
34 llvm-cxxdump verify-uselistorder dsymutil llvm-pdbdump
34 llvm-cxxdump verify-uselistorder dsymutil llvm-pdbdump \
35 llvm-split
3536
3637 # If Intel JIT Events support is configured, build an extra tool to test it.
3738 ifeq ($(USE_INTEL_JITEVENTS), 1)
0 set(LLVM_LINK_COMPONENTS
1 TransformUtils
2 BitWriter
3 Core
4 IRReader
5 Support
6 )
7
8 add_llvm_tool(llvm-split
9 llvm-split.cpp
10 )
0 ;===- ./tools/llvm-split/LLVMBuild.txt -------------------------*- Conf -*--===;
1 ;
2 ; The LLVM Compiler Infrastructure
3 ;
4 ; This file is distributed under the University of Illinois Open Source
5 ; License. See LICENSE.TXT for details.
6 ;
7 ;===------------------------------------------------------------------------===;
8 ;
9 ; This is an LLVMBuild description file for the components in this subdirectory.
10 ;
11 ; For more information on the LLVMBuild system, please see:
12 ;
13 ; http://llvm.org/docs/LLVMBuild.html
14 ;
15 ;===------------------------------------------------------------------------===;
16
17 [component_0]
18 type = Tool
19 name = llvm-split
20 parent = Tools
21 required_libraries = TransformUtils BitWriter Core IRReader Support
0 ##===- tools/llvm-split/Makefile ---------------------------*- Makefile -*-===##
1 #
2 # The LLVM Compiler Infrastructure
3 #
4 # This file is distributed under the University of Illinois Open Source
5 # License. See LICENSE.TXT for details.
6 #
7 ##===----------------------------------------------------------------------===##
8
9 LEVEL := ../..
10 TOOLNAME := llvm-split
11 LINK_COMPONENTS := transformutils bitwriter core irreader support
12
13 # This tool has no plugins, optimize startup time.
14 TOOL_NO_EXPORTS := 1
15
16 include $(LEVEL)/Makefile.common
0 //===-- llvm-split: command line tool for testing module splitter ---------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This program can be used to test the llvm::SplitModule function.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ADT/StringExtras.h"
14 #include "llvm/Bitcode/ReaderWriter.h"
15 #include "llvm/IR/LLVMContext.h"
16 #include "llvm/IRReader/IRReader.h"
17 #include "llvm/Support/CommandLine.h"
18 #include "llvm/Support/FileSystem.h"
19 #include "llvm/Support/SourceMgr.h"
20 #include "llvm/Support/ToolOutputFile.h"
21 #include "llvm/Support/raw_ostream.h"
22 #include "llvm/Transforms/Utils/SplitModule.h"
23
24 using namespace llvm;
25
26 static cl::opt
27 InputFilename(cl::Positional, cl::desc(""),
28 cl::init("-"), cl::value_desc("filename"));
29
30 static cl::opt
31 OutputFilename("o", cl::desc("Override output filename"),
32 cl::value_desc("filename"));
33
34 static cl::opt NumOutputs("j", cl::Prefix, cl::init(2),
35 cl::desc("Number of output files"));
36
37 int main(int argc, char **argv) {
38 LLVMContext &Context = getGlobalContext();
39 SMDiagnostic Err;
40 cl::ParseCommandLineOptions(argc, argv, "LLVM module splitter\n");
41
42 std::unique_ptr M = parseIRFile(InputFilename, Err, Context);
43
44 if (!M) {
45 Err.print(argv[0], errs());
46 return 1;
47 }
48
49 unsigned I = 0;
50 SplitModule(std::move(M), NumOutputs, [&](std::unique_ptr MPart) {
51 std::error_code EC;
52 std::unique_ptr Out(new tool_output_file(
53 OutputFilename + utostr(I++), EC, sys::fs::F_None));
54 if (EC) {
55 errs() << EC.message() << '\n';
56 exit(1);
57 }
58
59 WriteBitcodeToFile(MPart.get(), Out->os());
60
61 // Declare success.
62 Out->keep();
63 });
64
65 return 0;
66 }