llvm.org GIT mirror llvm / 21ddb43
[JITLink] Switch from an atom-based model to a "blocks and symbols" model. In the Atom model the symbols, content and relocations of a relocatable object file are represented as a graph of atoms, where each Atom represents a contiguous block of content with a single name (or no name at all if the content is anonymous), and where edges between Atoms represent relocations. If more than one symbol is associated with a contiguous block of content then the content is broken into multiple atoms and layout constraints (represented by edges) are introduced to ensure that the content remains effectively contiguous. These layout constraints must be kept in mind when examining the content associated with a symbol (it may be spread over multiple atoms) or when applying certain relocation types (e.g. MachO subtractors). This patch replaces the Atom model in JITLink with a blocks-and-symbols model. The blocks-and-symbols model represents relocatable object files as bipartite graphs, with one set of nodes representing contiguous content (Blocks) and another representing named or anonymous locations (Symbols) within a Block. Relocations are represented as edges from Blocks to Symbols. This scheme removes layout constraints (simplifying handling of MachO alt-entry symbols, and hopefully ELF sections at some point in the future) and simplifies some relocation logic. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@373689 91177308-0d34-0410-b5e6-96231b3b80d8 Lang Hames 1 year, 2 months ago
25 changed file(s) with 2605 addition(s) and 2320 deletion(s). Raw diff Collapse all Expand all
8080 /// Authors of JITLinkContexts can use this function to register a post-fixup
8181 /// pass that records the range of the eh-frame section. This range can
8282 /// be used after finalization to register and deregister the frame.
83 AtomGraphPassFunction
83 LinkGraphPassFunction
8484 createEHFrameRecorderPass(const Triple &TT,
8585 StoreFrameRangeFunction StoreFrameRange);
8686
3333 namespace llvm {
3434 namespace jitlink {
3535
36 class Symbol;
37 class Section;
38
3639 /// Base class for errors originating in JIT linker, e.g. missing relocation
3740 /// support.
3841 class JITLinkError : public ErrorInfo {
4952 std::string ErrMsg;
5053 };
5154
52 // Forward declare the Atom class.
53 class Atom;
54
55 /// Edge class. Represents both object file relocations, as well as layout and
56 /// keep-alive constraints.
55 /// Represents fixups and constraints in the LinkGraph.
5756 class Edge {
5857 public:
5958 using Kind = uint8_t;
6059
61 using GenericEdgeKind = enum : Kind {
60 enum GenericEdgeKind : Kind {
6261 Invalid, // Invalid edge value.
6362 FirstKeepAlive, // Keeps target alive. Offset/addend zero.
6463 KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness.
65 LayoutNext, // Layout constraint. Offset/Addend zero.
6664 FirstRelocation // First architecture specific relocation.
6765 };
6866
6967 using OffsetT = uint32_t;
7068 using AddendT = int64_t;
7169
72 Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend)
70 Edge(Kind K, OffsetT Offset, Symbol &Target, AddendT Addend)
7371 : Target(&Target), Offset(Offset), Addend(Addend), K(K) {}
7472
7573 OffsetT getOffset() const { return Offset; }
8179 return K - FirstRelocation;
8280 }
8381 bool isKeepAlive() const { return K >= FirstKeepAlive; }
84 Atom &getTarget() const { return *Target; }
85 void setTarget(Atom &Target) { this->Target = &Target; }
82 Symbol &getTarget() const { return *Target; }
83 void setTarget(Symbol &Target) { this->Target = &Target; }
8684 AddendT getAddend() const { return Addend; }
8785 void setAddend(AddendT Addend) { this->Addend = Addend; }
8886
8987 private:
90 Atom *Target;
91 OffsetT Offset;
92 AddendT Addend;
88 Symbol *Target = nullptr;
89 OffsetT Offset = 0;
90 AddendT Addend = 0;
9391 Kind K = 0;
9492 };
9593
96 using EdgeVector = std::vector;
97
98 const StringRef getGenericEdgeKindName(Edge::Kind K);
99
100 /// Base Atom class. Used by absolute and undefined atoms.
101 class Atom {
102 friend class AtomGraph;
94 /// Returns the string name of the given generic edge kind, or "unknown"
95 /// otherwise. Useful for debugging.
96 const char *getGenericEdgeKindName(Edge::Kind K);
97
98 /// Base class for Addressable entities (externals, absolutes, blocks).
99 class Addressable {
100 friend class LinkGraph;
103101
104102 protected:
105 /// Create a named (as yet unresolved) atom.
106 Atom(StringRef Name)
107 : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false),
108 IsGlobal(false), IsAbsolute(false), IsCallable(false),
109 IsExported(false), IsWeak(false), HasLayoutNext(false),
110 IsCommon(false) {}
111
112 /// Create an absolute symbol atom.
113 Atom(StringRef Name, JITTargetAddress Address)
114 : Name(Name), Address(Address), IsDefined(true), IsLive(false),
115 ShouldDiscard(false), IsGlobal(false), IsAbsolute(false),
116 IsCallable(false), IsExported(false), IsWeak(false),
117 HasLayoutNext(false), IsCommon(false) {}
103 Addressable(JITTargetAddress Address, bool IsDefined)
104 : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {}
105
106 Addressable(JITTargetAddress Address)
107 : Address(Address), IsDefined(false), IsAbsolute(true) {
108 assert(!(IsDefined && IsAbsolute) &&
109 "Block cannot be both defined and absolute");
110 }
118111
119112 public:
120 /// Returns true if this atom has a name.
121 bool hasName() const { return Name != StringRef(); }
122
123 /// Returns the name of this atom.
113 Addressable(const Addressable &) = delete;
114 Addressable &operator=(const Addressable &) = default;
115 Addressable(Addressable &&) = delete;
116 Addressable &operator=(Addressable &&) = default;
117
118 JITTargetAddress getAddress() const { return Address; }
119 void setAddress(JITTargetAddress Address) { this->Address = Address; }
120
121 /// Returns true if this is a defined addressable, in which case you
122 /// can downcast this to a .
123 bool isDefined() const { return static_cast(IsDefined); }
124 bool isAbsolute() const { return static_cast(IsAbsolute); }
125
126 private:
127 JITTargetAddress Address = 0;
128 uint64_t IsDefined : 1;
129 uint64_t IsAbsolute : 1;
130 };
131
132 using BlockOrdinal = unsigned;
133 using SectionOrdinal = unsigned;
134
135 /// An Addressable with content and edges.
136 class Block : public Addressable {
137 friend class LinkGraph;
138
139 private:
140 /// Create a zero-fill defined addressable.
141 Block(Section &Parent, BlockOrdinal Ordinal, JITTargetAddress Size,
142 JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset)
143 : Addressable(Address, true), Parent(Parent), Size(Size),
144 Ordinal(Ordinal) {
145 assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2");
146 assert(AlignmentOffset < Alignment &&
147 "Alignment offset cannot exceed alignment");
148 assert(AlignmentOffset <= MaxAlignmentOffset &&
149 "Alignment offset exceeds maximum");
150 P2Align = Alignment ? countTrailingZeros(Alignment) : 0;
151 this->AlignmentOffset = AlignmentOffset;
152 }
153
154 /// Create a defined addressable for the given content.
155 Block(Section &Parent, BlockOrdinal Ordinal, StringRef Content,
156 JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset)
157 : Addressable(Address, true), Parent(Parent), Data(Content.data()),
158 Size(Content.size()), Ordinal(Ordinal) {
159 assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2");
160 assert(AlignmentOffset < Alignment &&
161 "Alignment offset cannot exceed alignment");
162 assert(AlignmentOffset <= MaxAlignmentOffset &&
163 "Alignment offset exceeds maximum");
164 P2Align = Alignment ? countTrailingZeros(Alignment) : 0;
165 this->AlignmentOffset = AlignmentOffset;
166 }
167
168 public:
169 using EdgeVector = std::vector;
170 using edge_iterator = EdgeVector::iterator;
171 using const_edge_iterator = EdgeVector::const_iterator;
172
173 Block(const Block &) = delete;
174 Block &operator=(const Block &) = delete;
175 Block(Block &&) = delete;
176 Block &operator=(Block &&) = delete;
177
178 /// Return the parent section for this block.
179 Section &getSection() const { return Parent; }
180
181 /// Return the ordinal for this block.
182 BlockOrdinal getOrdinal() const { return Ordinal; }
183
184 /// Returns true if this is a zero-fill block.
185 ///
186 /// If true, getSize is callable but getContent is not (the content is
187 /// defined to be a sequence of zero bytes of length Size).
188 bool isZeroFill() const { return !Data; }
189
190 /// Returns the size of this defined addressable.
191 size_t getSize() const { return Size; }
192
193 /// Get the content for this block. Block must not be a zero-fill block.
194 StringRef getContent() const {
195 assert(Data && "Section does not contain content");
196 return StringRef(Data, Size);
197 }
198
199 /// Set the content for this block.
200 /// Caller is responsible for ensuring the underlying bytes are not
201 /// deallocated while pointed to by this block.
202 void setContent(StringRef Content) {
203 Data = Content.data();
204 Size = Content.size();
205 }
206
207 /// Get the alignment for this content.
208 uint64_t getAlignment() const { return 1 << P2Align; }
209
210 /// Get the alignment offset for this content.
211 uint64_t getAlignmentOffset() const { return AlignmentOffset; }
212
213 /// Add an edge to this block.
214 void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target,
215 Edge::AddendT Addend) {
216 Edges.push_back(Edge(K, Offset, Target, Addend));
217 }
218
219 /// Return the list of edges attached to this content.
220 iterator_range edges() {
221 return make_range(Edges.begin(), Edges.end());
222 }
223
224 /// Returns the list of edges attached to this content.
225 iterator_range edges() const {
226 return make_range(Edges.begin(), Edges.end());
227 }
228
229 /// Return the size of the edges list.
230 size_t edges_size() const { return Edges.size(); }
231
232 /// Returns true if the list of edges is empty.
233 bool edges_empty() const { return Edges.empty(); }
234
235 private:
236 static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1;
237
238 uint64_t P2Align : 5;
239 uint64_t AlignmentOffset : 57;
240 Section &Parent;
241 const char *Data = nullptr;
242 size_t Size = 0;
243 BlockOrdinal Ordinal = 0;
244 std::vector Edges;
245 };
246
247 /// Describes symbol linkage. This can be used to make resolve definition
248 /// clashes.
249 enum class Linkage : uint8_t {
250 Strong,
251 Weak,
252 };
253
254 /// For errors and debugging output.
255 const char *getLinkageName(Linkage L);
256
257 /// Defines the scope in which this symbol should be visible:
258 /// Default -- Visible in the public interface of the linkage unit.
259 /// Hidden -- Visible within the linkage unit, but not exported from it.
260 /// Local -- Visible only within the LinkGraph.
261 enum class Scope : uint8_t { Default, Hidden, Local };
262
263 /// For debugging output.
264 const char *getScopeName(Scope S);
265
266 raw_ostream &operator<<(raw_ostream &OS, const Block &B);
267
268 /// Symbol representation.
269 ///
270 /// Symbols represent locations within Addressable objects.
271 /// They can be either Named or Anonymous.
272 /// Anonymous symbols have neither linkage nor visibility, and must point at
273 /// ContentBlocks.
274 /// Named symbols may be in one of four states:
275 /// - Null: Default initialized. Assignable, but otherwise unusable.
276 /// - Defined: Has both linkage and visibility and points to a ContentBlock
277 /// - Common: Has both linkage and visibility, points to a null Addressable.
278 /// - External: Has neither linkage nor visibility, points to an external
279 /// Addressable.
280 ///
281 class Symbol {
282 friend class LinkGraph;
283
284 private:
285 Symbol(Addressable &Base, JITTargetAddress Offset, StringRef Name,
286 JITTargetAddress Size, Linkage L, Scope S, bool IsLive,
287 bool IsCallable)
288 : Name(Name), Base(&Base), Offset(Offset), Size(Size) {
289 setLinkage(L);
290 setScope(S);
291 setLive(IsLive);
292 setCallable(IsCallable);
293 }
294
295 static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name,
296 JITTargetAddress Size, Scope S, bool IsLive) {
297 assert(SymStorage && "Storage cannot be null");
298 assert(!Name.empty() && "Common symbol name cannot be empty");
299 assert(Base.isDefined() &&
300 "Cannot create common symbol from undefined block");
301 assert(static_cast(Base).getSize() == Size &&
302 "Common symbol size should match underlying block size");
303 auto *Sym = reinterpret_cast(SymStorage);
304 new (Sym) Symbol(Base, 0, Name, Size, Linkage::Weak, S, IsLive, false);
305 return *Sym;
306 }
307
308 static Symbol &constructExternal(void *SymStorage, Addressable &Base,
309 StringRef Name, JITTargetAddress Size) {
310 assert(SymStorage && "Storage cannot be null");
311 assert(!Base.isDefined() &&
312 "Cannot create external symbol from defined block");
313 assert(!Name.empty() && "External symbol name cannot be empty");
314 auto *Sym = reinterpret_cast(SymStorage);
315 new (Sym) Symbol(Base, 0, Name, Size, Linkage::Strong, Scope::Default,
316 false, false);
317 return *Sym;
318 }
319
320 static Symbol &constructAbsolute(void *SymStorage, Addressable &Base,
321 StringRef Name, JITTargetAddress Size,
322 Linkage L, Scope S, bool IsLive) {
323 assert(SymStorage && "Storage cannot be null");
324 assert(!Base.isDefined() &&
325 "Cannot create absolute symbol from a defined block");
326 auto *Sym = reinterpret_cast(SymStorage);
327 new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false);
328 return *Sym;
329 }
330
331 static Symbol &constructAnonDef(void *SymStorage, Block &Base,
332 JITTargetAddress Offset,
333 JITTargetAddress Size, bool IsCallable,
334 bool IsLive) {
335 assert(SymStorage && "Storage cannot be null");
336 auto *Sym = reinterpret_cast(SymStorage);
337 new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong,
338 Scope::Local, IsLive, IsCallable);
339 return *Sym;
340 }
341
342 static Symbol &constructNamedDef(void *SymStorage, Block &Base,
343 JITTargetAddress Offset, StringRef Name,
344 JITTargetAddress Size, Linkage L, Scope S,
345 bool IsLive, bool IsCallable) {
346 assert(SymStorage && "Storage cannot be null");
347 assert(!Name.empty() && "Name cannot be empty");
348 auto *Sym = reinterpret_cast(SymStorage);
349 new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable);
350 return *Sym;
351 }
352
353 public:
354 /// Create a null Symbol. This allows Symbols to be default initialized for
355 /// use in containers (e.g. as map values). Null symbols are only useful for
356 /// assigning to.
357 Symbol() = default;
358
359 // Symbols are not movable or copyable.
360 Symbol(const Symbol &) = delete;
361 Symbol &operator=(const Symbol &) = delete;
362 Symbol(Symbol &&) = delete;
363 Symbol &operator=(Symbol &&) = delete;
364
365 /// Returns true if this symbol has a name.
366 bool hasName() const { return !Name.empty(); }
367
368 /// Returns the name of this symbol (empty if the symbol is anonymous).
369 StringRef getName() const {
370 assert((!Name.empty() || getScope() == Scope::Local) &&
371 "Anonymous symbol has non-local scope");
372 return Name;
373 }
374
375 /// Returns true if this Symbol has content (potentially) defined within this
376 /// object file (i.e. is anything but an external or absolute symbol).
377 bool isDefined() const {
378 assert(Base && "Attempt to access null symbol");
379 return Base->isDefined();
380 }
381
382 /// Returns true if this symbol is live (i.e. should be treated as a root for
383 /// dead stripping).
384 bool isLive() const {
385 assert(Base && "Attempting to access null symbol");
386 return IsLive;
387 }
388
389 /// Set this symbol's live bit.
390 void setLive(bool IsLive) { this->IsLive = IsLive; }
391
392 /// Returns true is this symbol is callable.
393 bool isCallable() const { return IsCallable; }
394
395 /// Set this symbol's callable bit.
396 void setCallable(bool IsCallable) { this->IsCallable = IsCallable; }
397
398 /// Returns true if the underlying addressable is an unresolved external.
399 bool isExternal() const {
400 assert(Base && "Attempt to access null symbol");
401 return !Base->isDefined() && !Base->isAbsolute();
402 }
403
404 /// Returns true if the underlying addressable is an absolute symbol.
405 bool isAbsolute() const {
406 assert(Base && "Attempt to access null symbol");
407 return !Base->isDefined() && Base->isAbsolute();
408 }
409
410 /// Return the addressable that this symbol points to.
411 Addressable &getAddressable() {
412 assert(Base && "Cannot get underlying addressable for null symbol");
413 return *Base;
414 }
415
416 /// Return the addressable that thsi symbol points to.
417 const Addressable &getAddressable() const {
418 assert(Base && "Cannot get underlying addressable for null symbol");
419 return *Base;
420 }
421
422 /// Return the Block for this Symbol (Symbol must be defined).
423 Block &getBlock() {
424 assert(Base && "Cannot get block for null symbol");
425 assert(Base->isDefined() && "Not a defined symbol");
426 return static_cast(*Base);
427 }
428
429 /// Return the Block for this Symbol (Symbol must be defined).
430 const Block &getBlock() const {
431 assert(Base && "Cannot get block for null symbol");
432 assert(Base->isDefined() && "Not a defined symbol");
433 return static_cast(*Base);
434 }
435
436 /// Returns the offset for this symbol within the underlying addressable.
437 JITTargetAddress getOffset() const { return Offset; }
438
439 /// Returns the address of this symbol.
440 JITTargetAddress getAddress() const { return Base->getAddress() + Offset; }
441
442 /// Returns the size of this symbol.
443 JITTargetAddress getSize() const { return Size; }
444
445 /// Returns true if this symbol is backed by a zero-fill block.
446 /// This method may only be called on defined symbols.
447 bool isSymbolZeroFill() const { return getBlock().isZeroFill(); }
448
449 /// Returns the content in the underlying block covered by this symbol.
450 /// This method may only be called on defined non-zero-fill symbols.
451 StringRef getSymbolContent() const {
452 return getBlock().getContent().substr(Offset, Size);
453 }
454
455 /// Get the linkage for this Symbol.
456 Linkage getLinkage() const { return static_cast(L); }
457
458 /// Set the linkage for this Symbol.
459 void setLinkage(Linkage L) {
460 assert((L == Linkage::Strong || (Base->isDefined() && !Name.empty())) &&
461 "Linkage can only be applied to defined named symbols");
462 this->L = static_cast(L);
463 }
464
465 /// Get the visibility for this Symbol.
466 Scope getScope() const { return static_cast(S); }
467
468 /// Set the visibility for this Symbol.
469 void setScope(Scope S) {
470 assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) &&
471 "Invalid visibility for symbol type");
472 this->S = static_cast(S);
473 }
474
475 private:
476 void makeExternal(Addressable &A) {
477 assert(!A.isDefined() && "Attempting to make external with defined block");
478 Base = &A;
479 Offset = 0;
480 setLinkage(Linkage::Strong);
481 setScope(Scope::Default);
482 IsLive = 0;
483 // note: Size and IsCallable fields left unchanged.
484 }
485
486 static constexpr uint64_t MaxOffset = (1ULL << 59) - 1;
487
488 // FIXME: A char* or SymbolStringPtr may pack better.
489 StringRef Name;
490 Addressable *Base = nullptr;
491 uint64_t Offset : 59;
492 uint64_t L : 1;
493 uint64_t S : 2;
494 uint64_t IsLive : 1;
495 uint64_t IsCallable : 1;
496 JITTargetAddress Size = 0;
497 };
498
499 raw_ostream &operator<<(raw_ostream &OS, const Symbol &A);
500
501 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
502 StringRef EdgeKindName);
503
504 /// Represents an object file section.
505 class Section {
506 friend class LinkGraph;
507
508 private:
509 Section(StringRef Name, sys::Memory::ProtectionFlags Prot,
510 SectionOrdinal SecOrdinal)
511 : Name(Name), Prot(Prot), SecOrdinal(SecOrdinal) {}
512
513 using SymbolSet = DenseSet;
514 using BlockSet = DenseSet;
515
516 public:
517 using symbol_iterator = SymbolSet::iterator;
518 using const_symbol_iterator = SymbolSet::const_iterator;
519
520 using block_iterator = BlockSet::iterator;
521 using const_block_iterator = BlockSet::const_iterator;
522
523 ~Section();
524
525 /// Returns the name of this section.
124526 StringRef getName() const { return Name; }
125527
126 /// Returns the current target address of this atom.
127 /// The initial target address (for atoms that have one) will be taken from
128 /// the input object file's virtual address space. During the layout phase
129 /// of JIT linking the atom's address will be updated to point to its final
130 /// address in the JIT'd process.
131 JITTargetAddress getAddress() const { return Address; }
132
133 /// Set the current target address of this atom.
134 void setAddress(JITTargetAddress Address) { this->Address = Address; }
135
136 /// Returns true if this is a defined atom.
137 bool isDefined() const { return IsDefined; }
138
139 /// Returns true if this atom is marked as live.
140 bool isLive() const { return IsLive; }
141
142 /// Mark this atom as live.
143 ///
144 /// Note: Only defined and absolute atoms can be marked live.
145 void setLive(bool IsLive) {
146 assert((IsDefined || IsAbsolute || !IsLive) &&
147 "Only defined and absolute atoms can be marked live");
148 this->IsLive = IsLive;
149 }
150
151 /// Returns true if this atom should be discarded during pruning.
152 bool shouldDiscard() const { return ShouldDiscard; }
153
154 /// Mark this atom to be discarded.
155 ///
156 /// Note: Only defined and absolute atoms can be marked live.
157 void setShouldDiscard(bool ShouldDiscard) {
158 assert((IsDefined || IsAbsolute || !ShouldDiscard) &&
159 "Only defined and absolute atoms can be marked live");
160 this->ShouldDiscard = ShouldDiscard;
161 }
162
163 /// Returns true if this definition is global (i.e. visible outside this
164 /// linkage unit).
165 ///
166 /// Note: This is distict from Exported, which means visibile outside the
167 /// JITDylib that this graph is being linked in to.
168 bool isGlobal() const { return IsGlobal; }
169
170 /// Mark this atom as global.
171 void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; }
172
173 /// Returns true if this atom represents an absolute symbol.
174 bool isAbsolute() const { return IsAbsolute; }
175
176 /// Returns true if this atom is known to be callable.
177 ///
178 /// Primarily provided for easy interoperability with ORC, which uses the
179 /// JITSymbolFlags::Common flag to identify symbols that can be interposed
180 /// with stubs.
181 bool isCallable() const { return IsCallable; }
182
183 /// Mark this atom as callable.
184 void setCallable(bool IsCallable) {
185 assert((IsDefined || IsAbsolute || !IsCallable) &&
186 "Callable atoms must be defined or absolute");
187 this->IsCallable = IsCallable;
188 }
189
190 /// Returns true if this atom should appear in the symbol table of a final
191 /// linked image.
192 bool isExported() const { return IsExported; }
193
194 /// Mark this atom as exported.
195 void setExported(bool IsExported) {
196 assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) &&
197 "Exported atoms must have names");
198 this->IsExported = IsExported;
199 }
200
201 /// Returns true if this is a weak symbol.
202 bool isWeak() const { return IsWeak; }
203
204 /// Mark this atom as weak.
205 void setWeak(bool IsWeak) { this->IsWeak = IsWeak; }
206
207 private:
528 /// Returns the protection flags for this section.
529 sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; }
530
531 /// Returns the ordinal for this section.
532 SectionOrdinal getOrdinal() const { return SecOrdinal; }
533
534 /// Returns an iterator over the symbols defined in this section.
535 iterator_range symbols() {
536 return make_range(Symbols.begin(), Symbols.end());
537 }
538
539 /// Returns an iterator over the symbols defined in this section.
540 iterator_range symbols() const {
541 return make_range(Symbols.begin(), Symbols.end());
542 }
543
544 /// Return the number of symbols in this section.
545 SymbolSet::size_type symbols_size() { return Symbols.size(); }
546
547 /// Return true if this section contains no symbols.
548 bool symbols_empty() const { return Symbols.empty(); }
549
550 /// Returns the ordinal for the next block.
551 BlockOrdinal getNextBlockOrdinal() { return NextBlockOrdinal++; }
552
553 private:
554 void addSymbol(Symbol &Sym) {
555 assert(!Symbols.count(&Sym) && "Symbol is already in this section");
556 Symbols.insert(&Sym);
557 }
558
559 void removeSymbol(Symbol &Sym) {
560 assert(Symbols.count(&Sym) && "symbol is not in this section");
561 Symbols.erase(&Sym);
562 }
563
208564 StringRef Name;
209 JITTargetAddress Address = 0;
210
211 bool IsDefined : 1;
212 bool IsLive : 1;
213 bool ShouldDiscard : 1;
214
215 bool IsGlobal : 1;
216 bool IsAbsolute : 1;
217 bool IsCallable : 1;
218 bool IsExported : 1;
219 bool IsWeak : 1;
220
221 protected:
222 // These flags only make sense for DefinedAtom, but we can minimize the size
223 // of DefinedAtom by defining them here.
224 bool HasLayoutNext : 1;
225 bool IsCommon : 1;
226 };
227
228 // Forward declare DefinedAtom.
229 class DefinedAtom;
230
231 raw_ostream &operator<<(raw_ostream &OS, const Atom &A);
232 void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E,
233 StringRef EdgeKindName);
234
235 /// Represents a section address range via a pair of DefinedAtom pointers to
236 /// the first and last atoms in the section.
565 sys::Memory::ProtectionFlags Prot;
566 SectionOrdinal SecOrdinal = 0;
567 BlockOrdinal NextBlockOrdinal = 0;
568 SymbolSet Symbols;
569 };
570
571 /// Represents a section address range via a pair of Block pointers
572 /// to the first and last Blocks in the section.
237573 class SectionRange {
238574 public:
239575 SectionRange() = default;
240 SectionRange(DefinedAtom *First, DefinedAtom *Last)
241 : First(First), Last(Last) {}
242 DefinedAtom *getFirstAtom() const {
576 SectionRange(const Section &Sec) {
577 if (Sec.symbols_empty())
578 return;
579 First = Last = *Sec.symbols().begin();
580 for (auto *Sym : Sec.symbols()) {
581 if (Sym->getAddress() < First->getAddress())
582 First = Sym;
583 if (Sym->getAddress() > Last->getAddress())
584 Last = Sym;
585 }
586 }
587 Symbol *getFirstSymbol() const {
243588 assert((!Last || First) && "First can not be null if end is non-null");
244589 return First;
245590 }
246 DefinedAtom *getLastAtom() const {
591 Symbol *getLastSymbol() const {
247592 assert((First || !Last) && "Last can not be null if start is non-null");
248593 return Last;
249594 }
251596 assert((First || !Last) && "Last can not be null if start is non-null");
252597 return !First;
253598 }
254 JITTargetAddress getStart() const;
255 JITTargetAddress getEnd() const;
256 uint64_t getSize() const;
257
258 private:
259 DefinedAtom *First = nullptr;
260 DefinedAtom *Last = nullptr;
261 };
262
263 /// Represents an object file section.
264 class Section {
265 friend class AtomGraph;
266
267 private:
268 Section(StringRef Name, uint32_t Alignment, sys::Memory::ProtectionFlags Prot,
269 unsigned Ordinal, bool IsZeroFill)
270 : Name(Name), Alignment(Alignment), Prot(Prot), Ordinal(Ordinal),
271 IsZeroFill(IsZeroFill) {
272 assert(isPowerOf2_32(Alignment) && "Alignments must be a power of 2");
273 }
274
275 using DefinedAtomSet = DenseSet;
599 JITTargetAddress getStart() const {
600 return First ? First->getBlock().getAddress() : 0;
601 }
602 JITTargetAddress getEnd() const {
603 return Last ? Last->getBlock().getAddress() + Last->getBlock().getSize()
604 : 0;
605 }
606 uint64_t getSize() const { return getEnd() - getStart(); }
607
608 private:
609 Symbol *First = nullptr;
610 Symbol *Last = nullptr;
611 };
612
613 class LinkGraph {
614 private:
615 using SectionList = std::vector>;
616 using ExternalSymbolSet = DenseSet;
617 using BlockSet = DenseSet;
618
619 template
620 Addressable &createAddressable(ArgTs &&... Args) {
621 Addressable *A =
622 reinterpret_cast(Allocator.Allocate());
623 new (A) Addressable(std::forward(Args)...);
624 return *A;
625 }
626
627 void destroyAddressable(Addressable &A) {
628 A.~Addressable();
629 Allocator.Deallocate(&A);
630 }
631
632 template Block &createBlock(ArgTs &&... Args) {
633 Block *B = reinterpret_cast(Allocator.Allocate());
634 new (B) Block(std::forward(Args)...);
635 return *B;
636 }
637
638 void destroyBlock(Block &B) {
639 B.~Block();
640 Allocator.Deallocate(&B);
641 }
642
643 void destroySymbol(Symbol &S) {
644 S.~Symbol();
645 Allocator.Deallocate(&S);
646 }
276647
277648 public:
278 using atom_iterator = DefinedAtomSet::iterator;
279 using const_atom_iterator = DefinedAtomSet::const_iterator;
280
281 ~Section();
282 StringRef getName() const { return Name; }
283 uint32_t getAlignment() const { return Alignment; }
284 sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; }
285 unsigned getSectionOrdinal() const { return Ordinal; }
286 size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; }
287
288 bool isZeroFill() const { return IsZeroFill; }
289
290 /// Returns an iterator over the atoms in the section (in no particular
291 /// order).
292 iterator_range atoms() {
293 return make_range(DefinedAtoms.begin(), DefinedAtoms.end());
294 }
295
296 /// Returns an iterator over the atoms in the section (in no particular
297 /// order).
298 iterator_range atoms() const {
299 return make_range(DefinedAtoms.begin(), DefinedAtoms.end());
300 }
301
302 /// Return the number of atoms in this section.
303 DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); }
304
305 /// Return true if this section contains no atoms.
306 bool atoms_empty() const { return DefinedAtoms.empty(); }
307
308 /// Returns the range of this section as the pair of atoms with the lowest
309 /// and highest target address. This operation is expensive, as it
310 /// must traverse all atoms in the section.
311 ///
312 /// Note: If the section is empty, both values will be null. The section
313 /// address will evaluate to null, and the size to zero. If the section
314 /// contains a single atom both values will point to it, the address will
315 /// evaluate to the address of that atom, and the size will be the size of
316 /// that atom.
317 SectionRange getRange() const;
318
319 private:
320 void addAtom(DefinedAtom &DA) {
321 assert(!DefinedAtoms.count(&DA) && "Atom is already in this section");
322 DefinedAtoms.insert(&DA);
323 }
324
325 void removeAtom(DefinedAtom &DA) {
326 assert(DefinedAtoms.count(&DA) && "Atom is not in this section");
327 DefinedAtoms.erase(&DA);
328 }
329
330 StringRef Name;
331 uint32_t Alignment = 0;
332 sys::Memory::ProtectionFlags Prot;
333 unsigned Ordinal = 0;
334 unsigned NextAtomOrdinal = 0;
335 bool IsZeroFill = false;
336 DefinedAtomSet DefinedAtoms;
337 };
338
339 /// Defined atom class. Suitable for use by defined named and anonymous
340 /// atoms.
341 class DefinedAtom : public Atom {
342 friend class AtomGraph;
343
344 private:
345 DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment)
346 : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()),
347 Alignment(Alignment) {
348 assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two");
349 }
350
351 DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address,
352 uint32_t Alignment)
353 : Atom(Name, Address), Parent(Parent),
354 Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) {
355 assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two");
356 }
357
358 public:
359 using edge_iterator = EdgeVector::iterator;
360
361 Section &getSection() const { return Parent; }
362
363 uint64_t getSize() const { return Size; }
364
365 StringRef getContent() const {
366 assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom");
367 assert(Size <= std::numeric_limits::max() &&
368 "Content size too large");
369 return {ContentPtr, static_cast(Size)};
370 }
371 void setContent(StringRef Content) {
372 assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?");
373 ContentPtr = Content.data();
374 Size = Content.size();
375 }
376
377 bool isZeroFill() const { return Parent.isZeroFill(); }
378
379 void setZeroFill(uint64_t Size) {
380 assert(Parent.isZeroFill() && !ContentPtr &&
381 "Can't set zero-fill length of a non zero-fill atom");
382 this->Size = Size;
383 }
384
385 uint64_t getZeroFillSize() const {
386 assert(Parent.isZeroFill() &&
387 "Can't get zero-fill length of a non zero-fill atom");
388 return Size;
389 }
390
391 uint32_t getAlignment() const { return Alignment; }
392
393 bool hasLayoutNext() const { return HasLayoutNext; }
394 void setLayoutNext(DefinedAtom &Next) {
395 assert(!HasLayoutNext && "Atom already has layout-next constraint");
396 HasLayoutNext = true;
397 Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0));
398 }
399 DefinedAtom &getLayoutNext() {
400 assert(HasLayoutNext && "Atom does not have a layout-next constraint");
401 DefinedAtom *Next = nullptr;
402 for (auto &E : edges())
403 if (E.getKind() == Edge::LayoutNext) {
404 assert(E.getTarget().isDefined() &&
405 "layout-next target atom must be a defined atom");
406 Next = static_cast(&E.getTarget());
407 break;
408 }
409 assert(Next && "Missing LayoutNext edge");
410 return *Next;
411 }
412
413 bool isCommon() const { return IsCommon; }
414
415 void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target,
416 Edge::AddendT Addend) {
417 assert(K != Edge::LayoutNext &&
418 "Layout edges should be added via setLayoutNext");
419 Edges.push_back(Edge(K, Offset, Target, Addend));
420 }
421
422 iterator_range edges() {
423 return make_range(Edges.begin(), Edges.end());
424 }
425 size_t edges_size() const { return Edges.size(); }
426 bool edges_empty() const { return Edges.empty(); }
427
428 unsigned getOrdinal() const { return Ordinal; }
429
430 private:
431 void setCommon(uint64_t Size) {
432 assert(ContentPtr == 0 && "Atom already has content?");
433 IsCommon = true;
434 setZeroFill(Size);
435 }
436
437 EdgeVector Edges;
438 uint64_t Size = 0;
439 Section &Parent;
440 const char *ContentPtr = nullptr;
441 unsigned Ordinal = 0;
442 uint32_t Alignment = 0;
443 };
444
445 inline JITTargetAddress SectionRange::getStart() const {
446 return First ? First->getAddress() : 0;
447 }
448
449 inline JITTargetAddress SectionRange::getEnd() const {
450 return Last ? Last->getAddress() + Last->getSize() : 0;
451 }
452
453 inline uint64_t SectionRange::getSize() const { return getEnd() - getStart(); }
454
455 inline SectionRange Section::getRange() const {
456 if (atoms_empty())
457 return SectionRange();
458 DefinedAtom *First = *DefinedAtoms.begin(), *Last = *DefinedAtoms.begin();
459 for (auto *DA : atoms()) {
460 if (DA->getAddress() < First->getAddress())
461 First = DA;
462 if (DA->getAddress() > Last->getAddress())
463 Last = DA;
464 }
465 return SectionRange(First, Last);
466 }
467
468 class AtomGraph {
469 private:
470 using SectionList = std::vector>;
471 using AddressToAtomMap = std::map;
472 using NamedAtomMap = DenseMap;
473 using ExternalAtomSet = DenseSet;
474
475 public:
476 using external_atom_iterator = ExternalAtomSet::iterator;
649 using external_symbol_iterator = ExternalSymbolSet::iterator;
650
651 using block_iterator = BlockSet::iterator;
477652
478653 using section_iterator = pointee_iterator;
479654 using const_section_iterator = pointee_iterator;
480655
481 template
482 class defined_atom_iterator_impl
656 template
657 class defined_symbol_iterator_impl
483658 : public iterator_facade_base<
484 defined_atom_iterator_implItrT, T>,
659 defined_symbol_iterator_implItrT, T>,
485660 std::forward_iterator_tag, T> {
486661 public:
487 defined_atom_iterator_impl() = default;
488
489 defined_atom_iterator_impl(SecItrT SI, SecItrT SE)
490 : SI(SI), SE(SE),
491 AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) {
492 moveToNextAtomOrEnd();
662 defined_symbol_iterator_impl() = default;
663
664 defined_symbol_iterator_impl(SectionItrT SecI, SectionItrT SecE)
665 : SecI(SecI), SecE(SecE),
666 SymI(SecI != SecE ? SecI->symbols().begin() : SymbolItrT()) {
667 moveToNextSymbolOrEnd();
493668 }
494669
495 bool operator==(const defined_atom_iterator_impl &RHS) const {
496 return (SI == RHS.SI) && (AI == RHS.AI);
670 bool operator==(const defined_symbol_iterator_impl &RHS) const {
671 return (SecI == RHS.SecI) && (SymI == RHS.SymI);
497672 }
498673
499674 T operator*() const {
500 assert(AI != SI->atoms().end() && "Dereferencing end?");
501 return *AI;
675 assert(SymI != SecI->symbols().end() && "Dereferencing end?");
676 return *SymI;
502677 }
503678
504 defined_atom_iterator_impl operator++() {
505 ++AI;
506 moveToNextAtomOrEnd();
679 defined_symbol_iterator_impl operator++() {
680 ++SymI;
681 moveToNextSymbolOrEnd();
507682 return *this;
508683 }
509684
510685 private:
511 void moveToNextAtomOrEnd() {
512 while (SI != SE && AI == SI->atoms().end()) {
513 ++SI;
514 if (SI == SE)
515 AI = Section::atom_iterator();
516 else
517 AI = SI->atoms().begin();
686 void moveToNextSymbolOrEnd() {
687 while (SecI != SecE && SymI == SecI->symbols().end()) {
688 ++SecI;
689 SymI = SecI == SecE ? SymbolItrT() : SecI->symbols().begin();
518690 }
519691 }
520692
521 SecItrT SI, SE;
522 AtomItrT AI;
693 SectionItrT SecI, SecE;
694 SymbolItrT SymI;
523695 };
524696
525 using defined_atom_iterator =
526 defined_atom_iterator_impl
527 DefinedAtom *>;
528
529 using const_defined_atom_iterator =
530 defined_atom_iterator_impl
531 Section::const_atom_iterator,
532 const DefinedAtom *>;
533
534 AtomGraph(std::string Name, unsigned PointerSize,
697 using defined_symbol_iterator =
698 defined_symbol_iterator_impl
699 Section::symbol_iterator, Symbol *>;
700
701 using const_defined_symbol_iterator = defined_symbol_iterator_impl<
702 const_section_iterator, Section::const_symbol_iterator, const Symbol *>;
703
704 LinkGraph(std::string Name, unsigned PointerSize,
535705 support::endianness Endianness)
536706 : Name(std::move(Name)), PointerSize(PointerSize),
537707 Endianness(Endianness) {}
543713 /// Returns the pointer size for use in this graph.
544714 unsigned getPointerSize() const { return PointerSize; }
545715
546 /// Returns the endianness of atom-content in this graph.
716 /// Returns the endianness of content in this graph.
547717 support::endianness getEndianness() const { return Endianness; }
548718
549719 /// Create a section with the given name, protection flags, and alignment.
550 Section &createSection(StringRef Name, uint32_t Alignment,
551 sys::Memory::ProtectionFlags Prot, bool IsZeroFill) {
552 std::unique_ptr
Sec(
553 new Section(Name, Alignment, Prot, Sections.size(), IsZeroFill));
720 Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot) {
721 std::unique_ptr
Sec(new Section(Name, Prot, Sections.size()));
554722 Sections.push_back(std::move(Sec));
555723 return *Sections.back();
556724 }
557725
558 /// Add an external atom representing an undefined symbol in this graph.
559 Atom &addExternalAtom(StringRef Name) {
560 assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted");
561 Atom *A = reinterpret_cast(
562 AtomAllocator.Allocate(sizeof(Atom), alignof(Atom)));
563 new (A) Atom(Name);
564 ExternalAtoms.insert(A);
565 NamedAtoms[Name] = A;
566 return *A;
567 }
568
569 /// Add an external atom representing an absolute symbol.
570 Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) {
571 assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted");
572 Atom *A = reinterpret_cast(
573 AtomAllocator.Allocate(sizeof(Atom), alignof(Atom)));
574 new (A) Atom(Name, Addr);
575 AbsoluteAtoms.insert(A);
576 NamedAtoms[Name] = A;
577 return *A;
578 }
579
580 /// Add an anonymous defined atom to the graph.
581 ///
582 /// Anonymous atoms have content but no name. They must have an address.
583 DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address,
584 uint32_t Alignment) {
585 DefinedAtom *A = reinterpret_cast(
586 AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom)));
587 new (A) DefinedAtom(Parent, Address, Alignment);
588 Parent.addAtom(*A);
589 getAddrToAtomMap()[A->getAddress()] = A;
590 return *A;
591 }
592
593 /// Add a defined atom to the graph.
594 ///
595 /// Allocates and constructs a DefinedAtom instance with the given parent,
596 /// name, address, and alignment.
597 DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name,
598 JITTargetAddress Address, uint32_t Alignment) {
599 assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted");
600 DefinedAtom *A = reinterpret_cast(
601 AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom)));
602 new (A) DefinedAtom(Parent, Name, Address, Alignment);
603 Parent.addAtom(*A);
604 getAddrToAtomMap()[A->getAddress()] = A;
605 NamedAtoms[Name] = A;
606 return *A;
607 }
608
609 /// Add a common symbol atom to the graph.
610 ///
611 /// Adds a common-symbol atom to the graph with the given parent, name,
612 /// address, alignment and size.
613 DefinedAtom &addCommonAtom(Section &Parent, StringRef Name,
614 JITTargetAddress Address, uint32_t Alignment,
615 uint64_t Size) {
616 assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted");
617 DefinedAtom *A = reinterpret_cast(
618 AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom)));
619 new (A) DefinedAtom(Parent, Name, Address, Alignment);
620 A->setCommon(Size);
621 Parent.addAtom(*A);
622 NamedAtoms[Name] = A;
623 return *A;
726 /// Create a content block.
727 Block &createContentBlock(Section &Parent, StringRef Content,
728 uint64_t Address, uint64_t Alignment,
729 uint64_t AlignmentOffset) {
730 auto &B = createBlock(Parent, Parent.getNextBlockOrdinal(), Content,
731 Address, Alignment, AlignmentOffset);
732 Blocks.insert(&B);
733 return B;
734 }
735
736 /// Create a zero-fill block.
737 Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address,
738 uint64_t Alignment, uint64_t AlignmentOffset) {
739 auto &B = createBlock(Parent, Parent.getNextBlockOrdinal(), Size, Address,
740 Alignment, AlignmentOffset);
741 Blocks.insert(&B);
742 return B;
743 }
744
745 /// Add an external symbol.
746 /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose
747 /// size is not known, you should substitute '0'.
748 Symbol &addExternalSymbol(StringRef Name, uint64_t Size) {
749 auto &Sym = Symbol::constructExternal(
750 Allocator.Allocate(), createAddressable(0, false), Name, Size);
751 ExternalSymbols.insert(&Sym);
752 return Sym;
753 }
754
755 /// Add an absolute symbol.
756 Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address,
757 uint64_t Size, Linkage L, Scope S, bool IsLive) {
758 auto &Sym = Symbol::constructAbsolute(Allocator.Allocate(),
759 createAddressable(Address), Name,
760 Size, L, S, IsLive);
761 AbsoluteSymbols.insert(&Sym);
762 return Sym;
763 }
764
765 /// Convenience method for adding a weak zero-fill symbol.
766 Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section,
767 JITTargetAddress Address, uint64_t Size,
768 uint64_t Alignment, bool IsLive) {
769 auto &Sym = Symbol::constructCommon(
770 Allocator.Allocate(),
771 createBlock(Section, Section.getNextBlockOrdinal(), Address, Size,
772 Alignment, 0),
773 Name, Size, S, IsLive);
774 Section.addSymbol(Sym);
775 return Sym;
776 }
777
778 /// Add an anonymous symbol.
779 Symbol &addAnonymousSymbol(Block &Content, JITTargetAddress Offset,
780 JITTargetAddress Size, bool IsCallable,
781 bool IsLive) {
782 auto &Sym = Symbol::constructAnonDef(Allocator.Allocate(), Content,
783 Offset, Size, IsCallable, IsLive);
784 Content.getSection().addSymbol(Sym);
785 return Sym;
786 }
787
788 /// Add a named symbol.
789 Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset,
790 StringRef Name, JITTargetAddress Size, Linkage L,
791 Scope S, bool IsCallable, bool IsLive) {
792 auto &Sym =
793 Symbol::constructNamedDef(Allocator.Allocate(), Content, Offset,
794 Name, Size, L, S, IsLive, IsCallable);
795 Content.getSection().addSymbol(Sym);
796 return Sym;
624797 }
625798
626799 iterator_range sections() {
637810 return nullptr;
638811 }
639812
640 iterator_range external_atoms() {
641 return make_range(ExternalAtoms.begin(), ExternalAtoms.end());
642 }
643
644 iterator_range absolute_atoms() {
645 return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end());
646 }
647
648 iterator_range defined_atoms() {
649 return make_range(defined_atom_iterator(Sections.begin(), Sections.end()),
650 defined_atom_iterator(Sections.end(), Sections.end()));
651 }
652
653 iterator_range<const_defined_atom_iterator> defined_atoms() const {
813 iterator_range<external_symbol_iterator> external_symbols() {
814 return make_range(ExternalSymbols.begin(), ExternalSymbols.end());
815 }
816
817 iterator_range absolute_symbols() {
818 return make_range(AbsoluteSymbols.begin(), AbsoluteSymbols.end());
819 }
820
821 iterator_range defined_symbols() {
822 return make_range(defined_symbol_iterator(Sections.begin(), Sections.end()),
823 defined_symbol_iterator(Sections.end(), Sections.end()));
824 }
825
826 iterator_range defined_symbols() const {
654827 return make_range(
655 const_defined_atom_iterator(Sections.begin(), Sections.end()),
656 const_defined_atom_iterator(Sections.end(), Sections.end()));
657 }
658
659 /// Returns the atom with the given name, which must exist in this graph.
660 Atom &getAtomByName(StringRef Name) {
661 auto I = NamedAtoms.find(Name);
662 assert(I != NamedAtoms.end() && "Name not in NamedAtoms map");
663 return *I->second;
664 }
665
666 /// Returns the atom with the given name, which must exist in this graph and
667 /// be a DefinedAtom.
668 DefinedAtom &getDefinedAtomByName(StringRef Name) {
669 auto &A = getAtomByName(Name);
670 assert(A.isDefined() && "Atom is not a defined atom");
671 return static_cast(A);
672 }
673
674 /// Search for the given atom by name.
675 /// Returns the atom (if found) or an error (if no atom with this name
676 /// exists).
677 Expected findAtomByName(StringRef Name) {
678 auto I = NamedAtoms.find(Name);
679 if (I == NamedAtoms.end())
680 return make_error("No atom named " + Name);
681 return *I->second;
682 }
683
684 /// Search for the given defined atom by name.
685 /// Returns the defined atom (if found) or an error (if no atom with this
686 /// name exists, or if one exists but is not a defined atom).
687 Expected findDefinedAtomByName(StringRef Name) {
688 auto I = NamedAtoms.find(Name);
689 if (I == NamedAtoms.end())
690 return make_error("No atom named " + Name);
691 if (!I->second->isDefined())
692 return make_error("Atom " + Name +
693 " exists but is not a "
694 "defined atom");
695 return static_cast(*I->second);
696 }
697
698 /// Returns the atom covering the given address, or an error if no such atom
699 /// exists.
700 ///
701 /// Returns null if no atom exists at the given address.
702 DefinedAtom *getAtomByAddress(JITTargetAddress Address) {
703 refreshAddrToAtomCache();
704
705 // If there are no defined atoms, bail out early.
706 if (AddrToAtomCache->empty())
707 return nullptr;
708
709 // Find the atom *after* the given address.
710 auto I = AddrToAtomCache->upper_bound(Address);
711
712 // If this address falls before any known atom, bail out.
713 if (I == AddrToAtomCache->begin())
714 return nullptr;
715
716 // The atom we're looking for is the one before the atom we found.
717 --I;
718
719 // Otherwise range check the atom that was found.
720 assert(!I->second->getContent().empty() && "Atom content not set");
721 if (Address >= I->second->getAddress() + I->second->getContent().size())
722 return nullptr;
723
724 return I->second;
725 }
726
727 /// Like getAtomByAddress, but returns an Error if the given address is not
728 /// covered by an atom, rather than a null pointer.
729 Expected findAtomByAddress(JITTargetAddress Address) {
730 if (auto *DA = getAtomByAddress(Address))
731 return *DA;
732 return make_error("No atom at address " +
733 formatv("{0:x16}", Address));
734 }
735
736 // Remove the given external atom from the graph.
737 void removeExternalAtom(Atom &A) {
738 assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom");
739 assert(ExternalAtoms.count(&A) && "A is not in the external atoms set");
740 ExternalAtoms.erase(&A);
741 A.~Atom();
742 }
743
744 /// Remove the given absolute atom from the graph.
745 void removeAbsoluteAtom(Atom &A) {
746 assert(A.isAbsolute() && "A is not an absolute atom");
747 assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set");
748 AbsoluteAtoms.erase(&A);
749 A.~Atom();
750 }
751
752 /// Remove the given defined atom from the graph.
753 void removeDefinedAtom(DefinedAtom &DA) {
754 if (AddrToAtomCache) {
755 assert(AddrToAtomCache->count(DA.getAddress()) &&
756 "Cache exists, but does not contain atom");
757 AddrToAtomCache->erase(DA.getAddress());
828 const_defined_symbol_iterator(Sections.begin(), Sections.end()),
829 const_defined_symbol_iterator(Sections.end(), Sections.end()));
830 }
831
832 iterator_range blocks() {
833 return make_range(Blocks.begin(), Blocks.end());
834 }
835
836 /// Turn a defined symbol into an external one.
837 void makeExternal(Symbol &Sym) {
838 if (Sym.getAddressable().isAbsolute()) {
839 assert(AbsoluteSymbols.count(&Sym) &&
840 "Sym is not in the absolute symbols set");
841 AbsoluteSymbols.erase(&Sym);
842 } else {
843 assert(Sym.isDefined() && "Sym is not a defined symbol");
844 Section &Sec = Sym.getBlock().getSection();
845 Sec.removeSymbol(Sym);
758846 }
759 if (DA.hasName()) {
760 assert(NamedAtoms.count(DA.getName()) && "Named atom not in map");
761 NamedAtoms.erase(DA.getName());
762 }
763 DA.getSection().removeAtom(DA);
764 DA.~DefinedAtom();
765 }
766
767 /// Invalidate the atom-to-address map.
768 void invalidateAddrToAtomMap() { AddrToAtomCache = None; }
847 Sym.makeExternal(createAddressable(false));
848 ExternalSymbols.insert(&Sym);
849 }
850
851 /// Removes an external symbol. Also removes the underlying Addressable.
852 void removeExternalSymbol(Symbol &Sym) {
853 assert(!Sym.isDefined() && !Sym.isAbsolute() &&
854 "Sym is not an external symbol");
855 assert(ExternalSymbols.count(&Sym) && "Symbol is not in the externals set");
856 ExternalSymbols.erase(&Sym);
857 Addressable &Base = *Sym.Base;
858 destroySymbol(Sym);
859 destroyAddressable(Base);
860 }
861
862 /// Remove an absolute symbol. Also removes the underlying Addressable.
863 void removeAbsoluteSymbol(Symbol &Sym) {
864 assert(!Sym.isDefined() && Sym.isAbsolute() &&
865 "Sym is not an absolute symbol");
866 assert(AbsoluteSymbols.count(&Sym) &&
867 "Symbol is not in the absolute symbols set");
868 AbsoluteSymbols.erase(&Sym);
869 Addressable &Base = *Sym.Base;
870 destroySymbol(Sym);
871 destroyAddressable(Base);
872 }
873
874 /// Removes defined symbols. Does not remove the underlying block.
875 void removeDefinedSymbol(Symbol &Sym) {
876 assert(Sym.isDefined() && "Sym is not a defined symbol");
877 Sym.getBlock().getSection().removeSymbol(Sym);
878 destroySymbol(Sym);
879 }
880
881 /// Remove a block.
882 void removeBlock(Block &B) {
883 Blocks.erase(&B);
884 destroyBlock(B);
885 }
769886
770887 /// Dump the graph.
771888 ///
777894 std::function());
778895
779896 private:
780 AddressToAtomMap &getAddrToAtomMap() {
781 refreshAddrToAtomCache();
782 return *AddrToAtomCache;
783 }
784
785 const AddressToAtomMap &getAddrToAtomMap() const {
786 refreshAddrToAtomCache();
787 return *AddrToAtomCache;
788 }
789
790 void refreshAddrToAtomCache() const {
791 if (!AddrToAtomCache) {
792 AddrToAtomCache = AddressToAtomMap();
793 for (auto *DA : defined_atoms())
794 (*AddrToAtomCache)[DA->getAddress()] = const_cast(DA);
795 }
796 }
797
798 // Put the BumpPtrAllocator first so that we don't free any of the atoms in
799 // it until all of their destructors have been run.
800 BumpPtrAllocator AtomAllocator;
897 // Put the BumpPtrAllocator first so that we don't free any of the underlying
898 // memory until the Symbol/Addressable destructors have been run.
899 BumpPtrAllocator Allocator;
801900
802901 std::string Name;
803902 unsigned PointerSize;
804903 support::endianness Endianness;
904 BlockSet Blocks;
805905 SectionList Sections;
806 NamedAtomMap NamedAtoms;
807 ExternalAtomSet ExternalAtoms;
808 ExternalAtomSet AbsoluteAtoms;
809 mutable Optional AddrToAtomCache;
810 };
811
812 /// A function for mutating AtomGraphs.
813 using AtomGraphPassFunction = std::function;
814
815 /// A list of atom graph passes.
816 using AtomGraphPassList = std::vector;
817
818 /// An atom graph pass configuration, consisting of a list of pre-prune,
906 ExternalSymbolSet ExternalSymbols;
907 ExternalSymbolSet AbsoluteSymbols;
908 };
909
910 /// A function for mutating LinkGraphs.
911 using LinkGraphPassFunction = std::function;
912
913 /// A list of LinkGraph passes.
914 using LinkGraphPassList = std::vector;
915
916 /// An LinkGraph pass configuration, consisting of a list of pre-prune,
819917 /// post-prune, and post-fixup passes.
820918 struct PassConfiguration {
821919
822920 /// Pre-prune passes.
823921 ///
824922 /// These passes are called on the graph after it is built, and before any
825 /// atoms have been pruned.
923 /// symbols have been pruned.
826924 ///
827 /// Notable use cases: Marking atoms live or should-discard.
828 AtomGraphPassList PrePrunePasses;
925 /// Notable use cases: Marking symbols live or should-discard.
926 LinkGraphPassList PrePrunePasses;
829927
830928 /// Post-prune passes.
831929 ///
832 /// These passes are called on the graph after dead and should-discard atoms
833 /// have been removed, but before fixups are applied.
930 /// These passes are called on the graph after dead stripping, but before
931 /// fixups are applied.
834932 ///
835 /// Notable use cases: Building GOT, stub, and TLV atoms.
836 AtomGraphPassList PostPrunePasses;
933 /// Notable use cases: Building GOT, stub, and TLV symbols.
934 LinkGraphPassList PostPrunePasses;
837935
838936 /// Post-fixup passes.
839937 ///
840 /// These passes are called on the graph after atom contents has been copied
938 /// These passes are called on the graph after block contents has been copied
841939 /// to working memory, and fixups applied.
842940 ///
843941 /// Notable use cases: Testing and validation.
844 AtomGraphPassList PostFixupPasses;
942 LinkGraphPassList PostFixupPasses;
845943 };
846944
847945 /// A map of symbol names to resolved addresses.
848946 using AsyncLookupResult = DenseMap;
849947
850 /// A function to call with a resolved symbol map (See AsyncLookupResult) or an
851 /// error if resolution failed.
852 using JITLinkAsyncLookupContinuation =
853 std::function LR)>;
854
855 /// An asynchronous symbol lookup. Performs a search (possibly asynchronously)
856 /// for the given symbols, calling the given continuation with either the result
857 /// (if the lookup succeeds), or an error (if the lookup fails).
858 using JITLinkAsyncLookupFunction =
859 std::function &Symbols,
860 JITLinkAsyncLookupContinuation LookupContinuation)>;
948 /// A function object to call with a resolved symbol map (See AsyncLookupResult)
949 /// or an error if resolution failed.
950 class JITLinkAsyncLookupContinuation {
951 public:
952 virtual ~JITLinkAsyncLookupContinuation() {}
953 virtual void run(Expected LR) = 0;
954
955 private:
956 virtual void anchor();
957 };
958
959 /// Create a lookup continuation from a function object.
960 template
961 std::unique_ptr
962 createLookupContinuation(Continuation Cont) {
963
964 class Impl final : public JITLinkAsyncLookupContinuation {
965 public:
966 Impl(Continuation C) : C(std::move(C)) {}
967 void run(Expected LR) override { C(std::move(LR)); }
968
969 private:
970 Continuation C;
971 };
972
973 return std::make_unique(std::move(Cont));
974 };
861975
862976 /// Holds context for a single jitLink invocation.
863977 class JITLinkContext {
880994 /// lookup continutation which it must call with a result to continue the
881995 /// linking process.
882996 virtual void lookup(const DenseSet &Symbols,
883 JITLinkAsyncLookupContinuation LookupContinuation) = 0;
884
885 /// Called by JITLink once all defined atoms in the graph have been assigned
886 /// their final memory locations in the target process. At this point he
887 /// atom graph can be, inspected to build a symbol table however the atom
997 std::unique_ptr LC) = 0;
998
999 /// Called by JITLink once all defined symbols in the graph have been assigned
1000 /// their final memory locations in the target process. At this point the
1001 /// LinkGraph can be inspected to build a symbol table, however the block
8881002 /// content will not generally have been copied to the target location yet.
889 virtual void notifyResolved(AtomGraph &G) = 0;
1003 virtual void notifyResolved(LinkGraph &G) = 0;
8901004
8911005 /// Called by JITLink to notify the context that the object has been
8921006 /// finalized (i.e. emitted to memory and memory permissions set). If all of
9031017
9041018 /// Returns the mark-live pass to be used for this link. If no pass is
9051019 /// returned (the default) then the target-specific linker implementation will
906 /// choose a conservative default (usually marking all atoms live).
1020 /// choose a conservative default (usually marking all symbols live).
9071021 /// This function is only called if shouldAddDefaultTargetPasses returns true,
9081022 /// otherwise the JITContext is responsible for adding a mark-live pass in
9091023 /// modifyPassConfig.
910 virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const;
1024 virtual LinkGraphPassFunction getMarkLivePass(const Triple &TT) const;
9111025
9121026 /// Called by JITLink to modify the pass pipeline prior to linking.
9131027 /// The default version performs no modification.
9141028 virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config);
9151029 };
9161030
917 /// Marks all atoms in a graph live. This can be used as a default, conservative
918 /// mark-live implementation.
919 Error markAllAtomsLive(AtomGraph &G);
1031 /// Marks all symbols in a graph live. This can be used as a default,
1032 /// conservative mark-live implementation.
1033 Error markAllSymbolsLive(LinkGraph &G);
9201034
9211035 /// Basic JITLink implementation.
9221036 ///
3232 class SegmentRequest {
3333 public:
3434 SegmentRequest() = default;
35 SegmentRequest(size_t ContentSize, unsigned ContentAlign,
36 uint64_t ZeroFillSize, unsigned ZeroFillAlign)
37 : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize),
38 ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {}
35 SegmentRequest(uint64_t Alignment, size_t ContentSize,
36 uint64_t ZeroFillSize)
37 : Alignment(Alignment), ContentSize(ContentSize),
38 ZeroFillSize(ZeroFillSize) {
39 assert(isPowerOf2_32(Alignment) && "Alignment must be power of 2");
40 }
41 uint64_t getAlignment() const { return Alignment; }
3942 size_t getContentSize() const { return ContentSize; }
40 unsigned getContentAlignment() const { return ContentAlign; }
4143 uint64_t getZeroFillSize() const { return ZeroFillSize; }
42 unsigned getZeroFillAlignment() const { return ZeroFillAlign; }
43
4444 private:
45 uint64_t Alignment = 0;
4546 size_t ContentSize = 0;
4647 uint64_t ZeroFillSize = 0;
47 unsigned ContentAlign = 0;
48 unsigned ZeroFillAlign = 0;
4948 };
5049
5150 using SegmentsRequestMap = DenseMap;
1919
2020 template class BasicGOTAndStubsBuilder {
2121 public:
22 BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {}
22 BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {}
2323
2424 void run() {
25 // We're going to be adding new atoms, but we don't want to iterate over
26 // the newly added ones, so just copy the existing atoms out.
27 std::vector DAs(G.defined_atoms().begin(),
28 G.defined_atoms().end());
25 // We're going to be adding new blocks, but we don't want to iterate over
26 // the newly added ones, so just copy the existing blocks out.
27 std::vector Blocks(G.blocks().begin(), G.blocks().end());
2928
30 for (auto *DA : DAs)
31 for (auto &E : DA->edges())
29 for (auto *B : Blocks)
30 for (auto &E : B->edges())
3231 if (impl().isGOTEdge(E))
33 impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget()));
32 impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget()));
3433 else if (impl().isExternalBranchEdge(E))
35 impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget()));
34 impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget()));
3635 }
3736
3837 protected:
39 Atom &getGOTEntryAtom(Atom &Target) {
38 Symbol &getGOTEntrySymbol(Symbol &Target) {
4039 assert(Target.hasName() && "GOT edge cannot point to anonymous target");
4140
4241 auto GOTEntryI = GOTEntries.find(Target.getName());
4847 GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first;
4948 }
5049
51 assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom");
50 assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol");
5251 return *GOTEntryI->second;
5352 }
5453
55 Atom &getStubAtom(Atom &Target) {
54 Symbol &getStubSymbol(Symbol &Target) {
5655 assert(Target.hasName() &&
5756 "External branch edge can not point to an anonymous target");
5857 auto StubI = Stubs.find(Target.getName());
5958
6059 if (StubI == Stubs.end()) {
61 auto &StubAtom = impl().createStub(Target);
62 StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first;
60 auto &StubSymbol = impl().createStub(Target);
61 StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first;
6362 }
6463
65 assert(StubI != Stubs.end() && "Count not get stub atom");
64 assert(StubI != Stubs.end() && "Count not get stub symbol");
6665 return *StubI->second;
6766 }
6867
69 AtomGraph &G;
68 LinkGraph &G;
7069
7170 private:
7271 BuilderImpl &impl() { return static_cast(*this); }
7372
74 DenseMap GOTEntries;
75 DenseMapDefinedAtom *> Stubs;
73 DenseMapSymbol *> GOTEntries;
74 DenseMap Stubs;
7675 };
7776
7877 } // end namespace jitlink
44 EHFrameSupport.cpp
55 MachO.cpp
66 MachO_x86_64.cpp
7 MachOAtomGraphBuilder.cpp
7 MachOLinkGraphBuilder.cpp
88
99 DEPENDS
1010 intrinsics_gen
1616 namespace llvm {
1717 namespace jitlink {
1818
19 EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection,
20 StringRef EHFrameContent,
21 JITTargetAddress EHFrameAddress,
22 Edge::Kind FDEToCIERelocKind,
23 Edge::Kind FDEToTargetRelocKind)
24 : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent),
25 EHFrameAddress(EHFrameAddress),
26 EHFrameReader(EHFrameContent, G.getEndianness()),
27 FDEToCIERelocKind(FDEToCIERelocKind),
28 FDEToTargetRelocKind(FDEToTargetRelocKind) {}
29
30 Error EHFrameParser::atomize() {
19 EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress,
20 StringRef EHFrameContent,
21 unsigned PointerSize,
22 support::endianness Endianness)
23 : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent),
24 PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {}
25
26 Error EHFrameBinaryParser::addToGraph() {
3127 while (!EHFrameReader.empty()) {
3228 size_t RecordOffset = EHFrameReader.getOffset();
29 JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset;
3330
3431 LLVM_DEBUG({
3532 dbgs() << "Processing eh-frame record at "
36 << format("0x%016" PRIx64, EHFrameAddress + RecordOffset)
37 << " (offset " << RecordOffset << ")\n";
33 << format("0x%016" PRIx64, RecordAddress) << " (offset "
34 << RecordOffset << ")\n";
3835 });
3936
40 size_t CIELength = 0;
41 uint32_t CIELengthField;
42 if (auto Err = EHFrameReader.readInteger(CIELengthField))
37 size_t RecordLength = 0;
38 uint32_t RecordLengthField;
39 if (auto Err = EHFrameReader.readInteger(RecordLengthField))
4340 return Err;
4441
45 // Process CIE length/extended-length fields to build the atom.
42 // Process CIE/FDE length/extended-length fields to build the blocks.
4643 //
4744 // The value of these fields describe the length of the *rest* of the CIE
4845 // (not including data up to the end of the field itself) so we have to
49 // bump CIELength to include the data up to the end of the field: 4 bytes
46 // bump RecordLength to include the data up to the end of the field: 4 bytes
5047 // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength.
51 if (CIELengthField == 0) // Length 0 means end of __eh_frame section.
48 if (RecordLengthField == 0) // Length 0 means end of __eh_frame section.
5249 break;
5350
5451 // If the regular length field's value is 0xffffffff, use extended length.
55 if (CIELengthField == 0xffffffff) {
56 uint64_t CIEExtendedLengthField;
57 if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField))
58 return Err;
59 if (CIEExtendedLengthField > EHFrameReader.bytesRemaining())
52 if (RecordLengthField == 0xffffffff) {
53 uint64_t ExtendedLengthField;
54 if (auto Err = EHFrameReader.readInteger(ExtendedLengthField))
55 return Err;
56 if (ExtendedLengthField > EHFrameReader.bytesRemaining())
6057 return make_error("CIE record extends past the end of "
6158 "the __eh_frame section");
62 if (CIEExtendedLengthField + 12 > std::numeric_limits::max())
59 if (ExtendedLengthField + 12 > std::numeric_limits::max())
6360 return make_error("CIE record too large to process");
64 CIELength = CIEExtendedLengthField + 12;
61 RecordLength = ExtendedLengthField + 12;
6562 } else {
66 if (CIELengthField > EHFrameReader.bytesRemaining())
63 if (RecordLengthField > EHFrameReader.bytesRemaining())
6764 return make_error("CIE record extends past the end of "
6865 "the __eh_frame section");
69 CIELength = CIELengthField + 4;
70 }
71
72 LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n");
73
74 // Add an atom for this record.
75 CurRecordAtom = &G.addAnonymousAtom(
76 EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize());
77 CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength));
66 RecordLength = RecordLengthField + 4;
67 }
68
69 LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n");
7870
7971 // Read the CIE Pointer.
8072 size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset();
8476
8577 // Based on the CIE pointer value, parse this as a CIE or FDE record.
8678 if (CIEPointer == 0) {
87 if (auto Err = processCIE())
79 if (auto Err = processCIE(RecordOffset, RecordLength))
8880 return Err;
8981 } else {
90 if (auto Err = processFDE(CIEPointerAddress, CIEPointer))
91 return Err;
92 }
93
94 EHFrameReader.setOffset(RecordOffset + CIELength);
82 if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress,
83 CIEPointer))
84 return Err;
85 }
86
87 EHFrameReader.setOffset(RecordOffset + RecordLength);
9588 }
9689
9790 return Error::success();
9891 }
9992
100 Expected
101 EHFrameParser::parseAugmentationString() {
93 void EHFrameBinaryParser::anchor() {}
94
95 Expected
96 EHFrameBinaryParser::parseAugmentationString() {
10297 AugmentationInfo AugInfo;
10398 uint8_t NextChar;
10499 uint8_t *NextField = &AugInfo.Fields[0];
138133 return std::move(AugInfo);
139134 }
140135
141 Expected EHFrameParser::readAbsolutePointer() {
136 Expected EHFrameBinaryParser::readAbsolutePointer() {
142137 static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t),
143138 "Result must be able to hold a uint64_t");
144139 JITTargetAddress Addr;
145 if (G.getPointerSize() == 8) {
140 if (PointerSize == 8) {
146141 if (auto Err = EHFrameReader.readInteger(Addr))
147142 return std::move(Err);
148 } else if (G.getPointerSize() == 4) {
143 } else if (PointerSize == 4) {
149144 uint32_t Addr32;
150145 if (auto Err = EHFrameReader.readInteger(Addr32))
151146 return std::move(Err);
155150 return Addr;
156151 }
157152
158 Error EHFrameParser::processCIE() {
153 Error EHFrameBinaryParser::processCIE(size_t RecordOffset,
154 size_t RecordLength) {
159155 // Use the dwarf namespace for convenient access to pointer encoding
160156 // constants.
161157 using namespace dwarf;
162158
163159 LLVM_DEBUG(dbgs() << " Record is CIE\n");
164160
165 CIEInformation CIEInfo(*CurRecordAtom);
161 auto &CIESymbol =
162 createCIERecord(EHFrameAddress + RecordOffset,
163 EHFrameContent.substr(RecordOffset, RecordLength));
164
165 CIEInformation CIEInfo(CIESymbol);
166166
167167 uint8_t Version = 0;
168168 if (auto Err = EHFrameReader.readInteger(Version))
178178
179179 // Skip the EH Data field if present.
180180 if (AugInfo->EHDataFieldPresent)
181 if (auto Err = EHFrameReader.skip(G.getPointerSize()))
181 if (auto Err = EHFrameReader.skip(PointerSize))
182182 return Err;
183183
184184 // Read and sanity check the code alignment factor.
225225 return make_error(
226226 "Unsupported LSDA pointer encoding " +
227227 formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " +
228 formatv("{0:x16}", CurRecordAtom->getAddress()));
228 formatv("{0:x16}", CIESymbol.getAddress()));
229229 break;
230230 }
231231 case 'P': {
238238 "Unspported personality pointer "
239239 "encoding " +
240240 formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " +
241 formatv("{0:x16}", CurRecordAtom->getAddress()));
241 formatv("{0:x16}", CIESymbol.getAddress()));
242242 uint32_t PersonalityPointerAddress;
243243 if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress))
244244 return Err;
253253 "Unsupported FDE address pointer "
254254 "encoding " +
255255 formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " +
256 formatv("{0:x16}", CurRecordAtom->getAddress()));
256 formatv("{0:x16}", CIESymbol.getAddress()));
257257 break;
258258 }
259259 default:
266266 return make_error("Read past the end of the augmentation "
267267 "data while parsing fields");
268268
269 assert(!CIEInfos.count(CurRecordAtom->getAddress()) &&
269 assert(!CIEInfos.count(CIESymbol.getAddress()) &&
270270 "Multiple CIEs recorded at the same address?");
271 CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo);
271 CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo);
272272
273273 return Error::success();
274274 }
275275
276 Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress,
277 uint32_t CIEPointer) {
276 Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength,
277 JITTargetAddress CIEPointerAddress,
278 uint32_t CIEPointer) {
278279 LLVM_DEBUG(dbgs() << " Record is FDE\n");
279280
280281 LLVM_DEBUG({
285286 auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer);
286287 if (CIEInfoItr == CIEInfos.end())
287288 return make_error(
288 "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) +
289 "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) +
289290 " points to non-existant CIE at " +
290291 formatv("{0:x16}", CIEPointerAddress - CIEPointer));
291292 auto &CIEInfo = CIEInfoItr->second;
292293
293 // The CIEPointer looks good. Add a relocation.
294 CurRecordAtom->addEdge(FDEToCIERelocKind,
295 CIEPointerAddress - CurRecordAtom->getAddress(),
296 *CIEInfo.CIEAtom, 0);
297
298294 // Read and sanity check the PC-start pointer and size.
299295 JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset();
300296
304300
305301 JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta;
306302 LLVM_DEBUG({
307 dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n";
303 dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n";
308304 });
309305
310 auto *TargetAtom = G.getAtomByAddress(PCBegin);
311
312 if (!TargetAtom)
306 auto *TargetSymbol = getSymbolAtAddress(PCBegin);
307
308 if (!TargetSymbol)
313309 return make_error("FDE PC-begin " +
314310 formatv("{0:x16}", PCBegin) +
315 " does not point at atom");
316
317 if (TargetAtom->getAddress() != PCBegin)
311 " does not point at symbol");
312
313 if (TargetSymbol->getAddress() != PCBegin)
318314 return make_error(
319315 "FDE PC-begin " + formatv("{0:x16}", PCBegin) +
320 " does not point to start of atom at " +
321 formatv("{0:x16}", TargetAtom->getAddress()));
322
323 LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n");
324
325 // The PC-start pointer and size look good. Add relocations.
326 CurRecordAtom->addEdge(FDEToTargetRelocKind,
327 PCBeginAddress - CurRecordAtom->getAddress(),
328 *TargetAtom, 0);
329
330 // Add a keep-alive relocation from the function to the FDE to ensure it is
331 // not dead stripped.
332 TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0);
316 " does not point to start of symbol at " +
317 formatv("{0:x16}", TargetSymbol->getAddress()));
318
319 LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n");
333320
334321 // Skip over the PC range size field.
335 if (auto Err = EHFrameReader.skip(G.getPointerSize()))
322 if (auto Err = EHFrameReader.skip(PointerSize))
336323 return Err;
337324
325 Symbol *LSDASymbol = nullptr;
326 JITTargetAddress LSDAAddress = 0;
338327 if (CIEInfo.FDEsHaveLSDAField) {
339328 uint64_t AugmentationDataSize;
340329 if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize))
341330 return Err;
342 if (AugmentationDataSize != G.getPointerSize())
331 if (AugmentationDataSize != PointerSize)
343332 return make_error(
344333 "Unexpected FDE augmentation data size (expected " +
345 Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) +
346 ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()));
347 JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset();
334 Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) +
335 ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset));
336 LSDAAddress = EHFrameAddress + EHFrameReader.getOffset();
348337 auto LSDADelta = readAbsolutePointer();
349338 if (!LSDADelta)
350339 return LSDADelta.takeError();
351340
352341 JITTargetAddress LSDA = LSDAAddress + *LSDADelta;
353342
354 auto *LSDAAtom = G.getAtomByAddress(LSDA);
355
356 if (!LSDAAtom)
343 LSDASymbol = getSymbolAtAddress(LSDA);
344
345 if (!LSDASymbol)
357346 return make_error("FDE LSDA " + formatv("{0:x16}", LSDA) +
358 " does not point at atom");
359
360 if (LSDAAtom->getAddress() != LSDA)
347 " does not point at symbol");
348
349 if (LSDASymbol->getAddress() != LSDA)
361350 return make_error(
362351 "FDE LSDA " + formatv("{0:x16}", LSDA) +
363 " does not point to start of atom at " +
364 formatv("{0:x16}", LSDAAtom->getAddress()));
365
366 LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n");
367
368 // LSDA looks good. Add relocations.
369 CurRecordAtom->addEdge(FDEToTargetRelocKind,
370 LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom,
371 0);
372 }
373
374 return Error::success();
375 }
376
377 Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
378 StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
379 Edge::Kind FDEToCIERelocKind,
380 Edge::Kind FDEToTargetRelocKind) {
381 return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress,
382 FDEToCIERelocKind, FDEToTargetRelocKind)
383 .atomize();
352 " does not point to start of symbol at " +
353 formatv("{0:x16}", LSDASymbol->getAddress()));
354
355 LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n");
356 }
357
358 JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset;
359 auto FDESymbol = createFDERecord(
360 RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength),
361 *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol,
362 PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress);
363
364 return FDESymbol.takeError();
384365 }
385366
386367 // Determine whether we can register EH tables.
522503
523504 InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {}
524505
525 AtomGraphPassFunction
506 LinkGraphPassFunction
526507 createEHFrameRecorderPass(const Triple &TT,
527508 StoreFrameRangeFunction StoreRangeAddress) {
528509 const char *EHFrameSectionName = nullptr;
532513 EHFrameSectionName = ".eh_frame";
533514
534515 auto RecordEHFrame =
535 [EHFrameSectionName,
536 StoreFrameRange = std::move(StoreRangeAddress)](AtomGraph &G) -> Error {
537 // Search for a non-empty eh-frame and record the address of the first atom
538 // in it.
516 [EHFrameSectionName,
517 StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error {
518 // Search for a non-empty eh-frame and record the address of the first
519 // symbol in it.
539520 JITTargetAddress Addr = 0;
540521 size_t Size = 0;
541522 if (auto *S = G.findSectionByName(EHFrameSectionName)) {
542 auto R = S->getRange();
523 auto R = SectionRange(*S);
543524 Addr = R.getStart();
544525 Size = R.getSize();
545526 }
2020 namespace llvm {
2121 namespace jitlink {
2222
23 /// A generic parser for eh-frame sections.
23 /// A generic binary parser for eh-frame sections.
2424 ///
25 /// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and
26 /// FDEToTarget relocation kinds.
27 class EHFrameParser {
25 /// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph.
26 ///
27 /// This parser assumes that the user has already verified that the EH-frame's
28 /// address range does not overlap any other section/symbol, so that generated
29 /// CIE/FDE records do not overlap other sections/symbols.
30 class EHFrameBinaryParser {
2831 public:
29 EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent,
30 JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind,
31 Edge::Kind FDEToTargetRelocKind);
32 Error atomize();
32 EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent,
33 unsigned PointerSize, support::endianness Endianness);
34 virtual ~EHFrameBinaryParser() {}
35
36 Error addToGraph();
3337
3438 private:
39 virtual void anchor();
40 virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0;
41 virtual Symbol &createCIERecord(JITTargetAddress RecordAddr,
42 StringRef RecordContent) = 0;
43 virtual Expected
44 createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent,
45 Symbol &CIE, size_t CIEOffset, Symbol &Func,
46 size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0;
47
3548 struct AugmentationInfo {
3649 bool AugmentationDataPresent = false;
3750 bool EHDataFieldPresent = false;
4053
4154 Expected parseAugmentationString();
4255 Expected readAbsolutePointer();
43 Error processCIE();
44 Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer);
56 Error processCIE(size_t RecordOffset, size_t RecordLength);
57 Error processFDE(size_t RecordOffset, size_t RecordLength,
58 JITTargetAddress CIEPointerOffset, uint32_t CIEPointer);
4559
4660 struct CIEInformation {
4761 CIEInformation() = default;
48 CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {}
49 DefinedAtom *CIEAtom = nullptr;
62 CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {}
63 Symbol *CIESymbol = nullptr;
5064 bool FDEsHaveLSDAField = false;
5165 };
5266
53 AtomGraph &G;
54 Section &EHFrameSection;
67 JITTargetAddress EHFrameAddress;
5568 StringRef EHFrameContent;
56 JITTargetAddress EHFrameAddress;
69 unsigned PointerSize;
5770 BinaryStreamReader EHFrameReader;
58 DefinedAtom *CurRecordAtom = nullptr;
5971 DenseMap CIEInfos;
60 Edge::Kind FDEToCIERelocKind;
61 Edge::Kind FDEToTargetRelocKind;
6272 };
63
64 Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
65 StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
66 Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind);
6773
6874 } // end namespace jitlink
6975 } // end namespace llvm
5555 return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory);
5656 }
5757
58 const StringRef getGenericEdgeKindName(Edge::Kind K) {
58 const char *getGenericEdgeKindName(Edge::Kind K) {
5959 switch (K) {
6060 case Edge::Invalid:
6161 return "INVALID RELOCATION";
6262 case Edge::KeepAlive:
6363 return "Keep-Alive";
64 case Edge::LayoutNext:
65 return "Layout-Next";
6664 default:
6765 llvm_unreachable("Unrecognized relocation kind");
6866 }
6967 }
7068
71 raw_ostream &operator<<(raw_ostream &OS, const Atom &A) {
69 const char *getLinkageName(Linkage L) {
70 switch (L) {
71 case Linkage::Strong:
72 return "strong";
73 case Linkage::Weak:
74 return "weak";
75 }
76 }
77
78 const char *getScopeName(Scope S) {
79 switch (S) {
80 case Scope::Default:
81 return "default";
82 case Scope::Hidden:
83 return "hidden";
84 case Scope::Local:
85 return "local";
86 }
87 }
88
89 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
90 return OS << formatv("{0:x16}", B.getAddress()) << " -- "
91 << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": "
92 << (B.isZeroFill() ? "zero-fill" : "content")
93 << ", align = " << B.getAlignment()
94 << ", align-ofs = " << B.getAlignmentOffset()
95 << ", section = " << B.getSection().getName();
96 }
97
98 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
7299 OS << "<";
73 if (A.getName().empty())
74 OS << "anon@" << format("0x%016" PRIx64, A.getAddress());
100 if (Sym.getName().empty())
101 OS << "*anon*";
75102 else
76 OS << A.getName();
77 OS << " [";
78 if (A.isDefined()) {
79 auto &DA = static_cast(A);
80 OS << " section=" << DA.getSection().getName();
81 if (DA.isLive())
82 OS << " live";
83 if (DA.shouldDiscard())
84 OS << " should-discard";
85 } else
86 OS << " external";
87 OS << " ]>";
103 OS << Sym.getName();
104 OS << ": flags = ";
105 switch (Sym.getLinkage()) {
106 case Linkage::Strong:
107 OS << 'S';
108 break;
109 case Linkage::Weak:
110 OS << 'W';
111 break;
112 }
113 switch (Sym.getScope()) {
114 case Scope::Default:
115 OS << 'D';
116 break;
117 case Scope::Hidden:
118 OS << 'H';
119 break;
120 case Scope::Local:
121 OS << 'L';
122 break;
123 }
124 OS << (Sym.isLive() ? '+' : '-')
125 << ", size = " << formatv("{0:x8}", Sym.getSize())
126 << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " ("
127 << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + "
128 << formatv("{0:x8}", Sym.getOffset());
129 if (Sym.isDefined())
130 OS << " " << Sym.getBlock().getSection().getName();
131 OS << ")>";
88132 return OS;
89133 }
90134
91 void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E,
135 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
92136 StringRef EdgeKindName) {
93 OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset())
94 << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName
95 << " -> " << E.getTarget() << " + " << E.getAddend();
137 OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": "
138 << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- "
139 << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend();
96140 }
97141
98142 Section::~Section() {
99 for (auto *DA : DefinedAtoms)
100 DA->~DefinedAtom();
101 }
102
103 void AtomGraph::dump(raw_ostream &OS,
143 for (auto *Sym : Symbols)
144 Sym->~Symbol();
145 }
146
147 void LinkGraph::dump(raw_ostream &OS,
104148 std::function EdgeKindToName) {
105149 if (!EdgeKindToName)
106150 EdgeKindToName = [](Edge::Kind K) { return StringRef(); };
107151
108 OS << "Defined atoms:\n";
109 for (auto *DA : defined_atoms()) {
110 OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA
152 OS << "Symbols:\n";
153 for (auto *Sym : defined_symbols()) {
154 OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
111155 << "\n";
112 for (auto &E : DA->edges()) {
113 OS << " ";
114 StringRef EdgeName = (E.getKind() < Edge::FirstRelocation
115 ? getGenericEdgeKindName(E.getKind())
116 : EdgeKindToName(E.getKind()));
117
118 if (!EdgeName.empty())
119 printEdge(OS, *DA, E, EdgeName);
120 else {
121 auto EdgeNumberString = std::to_string(E.getKind());
122 printEdge(OS, *DA, E, EdgeNumberString);
156 if (Sym->isDefined()) {
157 for (auto &E : Sym->getBlock().edges()) {
158 OS << " ";
159 StringRef EdgeName = (E.getKind() < Edge::FirstRelocation
160 ? getGenericEdgeKindName(E.getKind())
161 : EdgeKindToName(E.getKind()));
162
163 if (!EdgeName.empty())
164 printEdge(OS, Sym->getBlock(), E, EdgeName);
165 else {
166 auto EdgeNumberString = std::to_string(E.getKind());
167 printEdge(OS, Sym->getBlock(), E, EdgeNumberString);
168 }
169 OS << "\n";
123170 }
124 OS << "\n";
125171 }
126172 }
127173
128 OS << "Absolute atoms:\n";
129 for (auto *A : absolute_atoms())
130 OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A
174 OS << "Absolute symbols:\n";
175 for (auto *Sym : absolute_symbols())
176 OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
131177 << "\n";
132178
133 OS << "External atoms:\n";
134 for (auto *A : external_atoms())
135 OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A
179 OS << "External symbols:\n";
180 for (auto *Sym : external_symbols())
181 OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
136182 << "\n";
137183 }
184
185 void JITLinkAsyncLookupContinuation::anchor() {}
138186
139187 JITLinkContext::~JITLinkContext() {}
140188
142190 return true;
143191 }
144192
145 AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
146 return AtomGraphPassFunction();
193 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
194 return LinkGraphPassFunction();
147195 }
148196
149197 Error JITLinkContext::modifyPassConfig(const Triple &TT,
151199 return Error::success();
152200 }
153201
154 Error markAllAtomsLive(AtomGraph &G) {
155 for (auto *DA : G.defined_atoms())
156 DA->setLive(true);
202 Error markAllSymbolsLive(LinkGraph &G) {
203 for (auto *Sym : G.defined_symbols())
204 Sym->setLive(true);
157205 return Error::success();
158206 }
159207
1010 //===----------------------------------------------------------------------===//
1111
1212 #include "JITLinkGeneric.h"
13 #include "EHFrameSupportImpl.h"
1413
1514 #include "llvm/Support/BinaryStreamReader.h"
1615 #include "llvm/Support/MemoryBuffer.h"
2423
2524 void JITLinkerBase::linkPhase1(std::unique_ptr Self) {
2625
27 // Build the atom graph.
26 // Build the link graph.
2827 if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer()))
2928 G = std::move(*GraphOrErr);
3029 else
3231 assert(G && "Graph should have been created by buildGraph above");
3332
3433 // Prune and optimize the graph.
35 if (auto Err = runPasses(Passes.PrePrunePasses, *G))
34 if (auto Err = runPasses(Passes.PrePrunePasses))
3635 return Ctx->notifyFailed(std::move(Err));
3736
3837 LLVM_DEBUG({
39 dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n";
38 dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n";
4039 dumpGraph(dbgs());
4140 });
4241
4342 prune(*G);
4443
4544 LLVM_DEBUG({
46 dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n";
45 dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n";
4746 dumpGraph(dbgs());
4847 });
4948
5049 // Run post-pruning passes.
51 if (auto Err = runPasses(Passes.PostPrunePasses, *G))
50 if (auto Err = runPasses(Passes.PostPrunePasses))
5251 return Ctx->notifyFailed(std::move(Err));
5352
54 // Sort atoms into segments.
55 layOutAtoms();
53 // Sort blocks into segments.
54 auto Layout = layOutBlocks();
5655
5756 // Allocate memory for segments.
5857 if (auto Err = allocateSegments(Layout))
5958 return Ctx->notifyFailed(std::move(Err));
6059
61 // Notify client that the defined atoms have been assigned addresses.
60 // Notify client that the defined symbols have been assigned addresses.
6261 Ctx->notifyResolved(*G);
6362
6463 auto ExternalSymbols = getExternalSymbolNames();
7372 // [Self=std::move(Self)](Expected Result) {
7473 // Self->linkPhase2(std::move(Self), std::move(Result));
7574 // });
76 //
77 // FIXME: Use move capture once we have c++14.
7875 auto *TmpCtx = Ctx.get();
79 auto *UnownedSelf = Self.release();
80 auto Phase2Continuation =
81 [UnownedSelf](Expected LookupResult) {
82 std::unique_ptr Self(UnownedSelf);
83 UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult));
84 };
85 TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation));
76 TmpCtx->lookup(std::move(ExternalSymbols),
77 createLookupContinuation(
78 [S = std::move(Self), L = std::move(Layout)](
79 Expected LookupResult) mutable {
80 auto &TmpSelf = *S;
81 TmpSelf.linkPhase2(std::move(S), std::move(LookupResult),
82 std::move(L));
83 }));
8684 }
8785
8886 void JITLinkerBase::linkPhase2(std::unique_ptr Self,
89 Expected LR) {
87 Expected LR,
88 SegmentLayoutMap Layout) {
9089 // If the lookup failed, bail out.
9190 if (!LR)
9291 return deallocateAndBailOut(LR.takeError());
9392
94 // Assign addresses to external atoms.
93 // Assign addresses to external addressables.
9594 applyLookupResult(*LR);
9695
9796 LLVM_DEBUG({
98 dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n";
97 dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n";
9998 dumpGraph(dbgs());
10099 });
101100
102 // Copy atom content to working memory and fix up.
103 if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc))
101 // Copy block content to working memory and fix up.
102 if (auto Err = copyAndFixUpBlocks(Layout, *Alloc))
104103 return deallocateAndBailOut(std::move(Err));
105104
106105 LLVM_DEBUG({
107 dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n";
106 dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n";
108107 dumpGraph(dbgs());
109108 });
110109
111 if (auto Err = runPasses(Passes.PostFixupPasses, *G))
110 if (auto Err = runPasses(Passes.PostFixupPasses))
112111 return deallocateAndBailOut(std::move(Err));
113112
114113 // FIXME: Use move capture once we have c++14.
127126 Ctx->notifyFinalized(std::move(Alloc));
128127 }
129128
130 Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) {
129 Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) {
131130 for (auto &P : Passes)
132 if (auto Err = P(G))
131 if (auto Err = P(*G))
133132 return Err;
134133 return Error::success();
135134 }
136135
137 void JITLinkerBase::layOutAtoms() {
138 // Group sections by protections, and whether or not they're zero-fill.
139 for (auto &S : G->sections()) {
140
141 // Skip empty sections.
142 if (S.atoms_empty())
143 continue;
144
145 auto &SL = Layout[S.getProtectionFlags()];
146 if (S.isZeroFill())
147 SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S));
136 JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() {
137
138 SegmentLayoutMap Layout;
139
140 /// Partition blocks based on permissions and content vs. zero-fill.
141 for (auto *B : G->blocks()) {
142 auto &SegLists = Layout[B->getSection().getProtectionFlags()];
143 if (!B->isZeroFill())
144 SegLists.ContentBlocks.push_back(B);
148145 else
149 SL.ContentSections.push_back(SegmentLayout::SectionLayout(S));
150 }
151
152 // Sort sections within the layout by ordinal.
153 {
154 auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS,
155 const SegmentLayout::SectionLayout &RHS) {
156 return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal();
146 SegLists.ZeroFillBlocks.push_back(B);
147 }
148
149 /// Sort blocks within each list.
150 for (auto &KV : Layout) {
151
152 auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
153 if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
154 return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
155 return LHS->getOrdinal() < RHS->getOrdinal();
157156 };
158 for (auto &KV : Layout) {
159 auto &SL = KV.second;
160 std::sort(SL.ContentSections.begin(), SL.ContentSections.end(),
161 CompareByOrdinal);
162 std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(),
163 CompareByOrdinal);
164 }
165 }
166
167 // Add atoms to the sections.
168 for (auto &KV : Layout) {
169 auto &SL = KV.second;
170 for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) {
171 for (auto &SI : *SIList) {
172 // First build the set of layout-heads (i.e. "heads" of layout-next
173 // chains) by copying the section atoms, then eliminating any that
174 // appear as layout-next targets.
175 DenseSet LayoutHeads;
176 for (auto *DA : SI.S->atoms())
177 LayoutHeads.insert(DA);
178
179 for (auto *DA : SI.S->atoms())
180 if (DA->hasLayoutNext())
181 LayoutHeads.erase(&DA->getLayoutNext());
182
183 // Next, sort the layout heads by address order.
184 std::vector OrderedLayoutHeads;
185 OrderedLayoutHeads.reserve(LayoutHeads.size());
186 for (auto *DA : LayoutHeads)
187 OrderedLayoutHeads.push_back(DA);
188
189 // Now sort the list of layout heads by address.
190 std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(),
191 [](const DefinedAtom *LHS, const DefinedAtom *RHS) {
192 return LHS->getAddress() < RHS->getAddress();
193 });
194
195 // Now populate the SI.Atoms field by appending each of the chains.
196 for (auto *DA : OrderedLayoutHeads) {
197 SI.Atoms.push_back(DA);
198 while (DA->hasLayoutNext()) {
199 auto &Next = DA->getLayoutNext();
200 SI.Atoms.push_back(&Next);
201 DA = &Next;
202 }
203 }
204 }
205 }
157
158 auto &SegLists = KV.second;
159 llvm::sort(SegLists.ContentBlocks, CompareBlocks);
160 llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks);
206161 }
207162
208163 LLVM_DEBUG({
212167 << static_cast(KV.first) << ":\n";
213168 auto &SL = KV.second;
214169 for (auto &SIEntry :
215 {std::make_pair(&SL.ContentSections, "content sections"),
216 std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) {
217 auto &SIList = *SIEntry.first;
170 {std::make_pair(&SL.ContentBlocks, "content block"),
171 std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) {
218172 dbgs() << " " << SIEntry.second << ":\n";
219 for (auto &SI : SIList) {
220 dbgs() << " " << SI.S->getName() << ":\n";
221 for (auto *DA : SI.Atoms)
222 dbgs() << " " << *DA << "\n";
223 }
173 for (auto *B : *SIEntry.first)
174 dbgs() << " " << *B << "\n";
224175 }
225176 }
226177 });
178
179 return Layout;
227180 }
228181
229182 Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
233186 JITLinkMemoryManager::SegmentsRequestMap Segments;
234187 for (auto &KV : Layout) {
235188 auto &Prot = KV.first;
236 auto &SegLayout = KV.second;
189 auto &SegLists = KV.second;
190
191 uint64_t SegAlign = 1;
237192
238193 // Calculate segment content size.
239194 size_t SegContentSize = 0;
240 uint32_t SegContentAlign = 1;
241 for (auto &SI : SegLayout.ContentSections) {
242 assert(!SI.S->atoms_empty() && "Sections in layout must not be empty");
243 assert(!SI.Atoms.empty() && "Section layouts must not be empty");
244
245 // Bump to section alignment before processing atoms.
246 SegContentSize = alignTo(SegContentSize, SI.S->getAlignment());
247 SegContentAlign = std::max(SegContentAlign, SI.S->getAlignment());
248
249 for (auto *DA : SI.Atoms) {
250 SegContentSize = alignTo(SegContentSize, DA->getAlignment());
251 SegContentSize += DA->getSize();
252 SegContentAlign = std::max(SegContentAlign, DA->getAlignment());
253 }
254 }
255
256 // Calculate segment zero-fill size.
257 uint64_t SegZeroFillSize = 0;
258 uint32_t SegZeroFillAlign = 1;
259
260 for (auto &SI : SegLayout.ZeroFillSections) {
261 assert(!SI.S->atoms_empty() && "Sections in layout must not be empty");
262 assert(!SI.Atoms.empty() && "Section layouts must not be empty");
263
264 // Bump to section alignment before processing atoms.
265 SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment());
266 SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment());
267
268 for (auto *DA : SI.Atoms) {
269 SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment());
270 SegZeroFillSize += DA->getSize();
271 SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment());
272 }
273 }
274
275 assert(isPowerOf2_32(SegContentAlign) &&
276 "Expected content alignment to be power of 2");
277 assert(isPowerOf2_32(SegZeroFillAlign) &&
278 "Expected zero-fill alignment to be power of 2");
279 // Round content alignment up to segment alignment.
280 SegContentAlign = std::max(SegContentAlign, SegZeroFillAlign);
281
282 Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize,
283 SegZeroFillAlign};
195 for (auto *B : SegLists.ContentBlocks) {
196 SegAlign = std::max(SegAlign, B->getAlignment());
197 SegContentSize = alignToBlock(SegContentSize, *B);
198 SegContentSize += B->getSize();
199 }
200
201 uint64_t SegZeroFillStart = SegContentSize;
202 uint64_t SegZeroFillEnd = SegZeroFillStart;
203
204 for (auto *B : SegLists.ZeroFillBlocks) {
205 SegAlign = std::max(SegAlign, B->getAlignment());
206 SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B);
207 SegZeroFillEnd += B->getSize();
208 }
209
210 Segments[Prot] = {SegAlign, SegContentSize,
211 SegZeroFillEnd - SegZeroFillStart};
284212
285213 LLVM_DEBUG({
286214 dbgs() << (&KV == &*Layout.begin() ? "" : "; ")
287 << static_cast(Prot) << ": "
288 << SegContentSize << " content bytes (alignment "
289 << SegContentAlign << ") + " << SegZeroFillSize
290 << " zero-fill bytes (alignment " << SegZeroFillAlign << ")";
215 << static_cast(Prot)
216 << ": alignment = " << SegAlign
217 << ", content size = " << SegContentSize
218 << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart);
291219 });
292220 }
293221 LLVM_DEBUG(dbgs() << " }\n");
306234 }
307235 });
308236
309 // Update atom target addresses.
237 // Update block target addresses.
310238 for (auto &KV : Layout) {
311239 auto &Prot = KV.first;
312240 auto &SL = KV.second;
313241
314 JITTargetAddress AtomTargetAddr =
242 JITTargetAddress NextBlockAddr =
315243 Alloc->getTargetMemory(static_cast(Prot));
316244
317 for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections})
318 for (auto &SI : *SIList) {
319 AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment());
320 for (auto *DA : SI.Atoms) {
321 AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment());
322 DA->setAddress(AtomTargetAddr);
323 AtomTargetAddr += DA->getSize();
324 }
245 for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks})
246 for (auto *B : *SIList) {
247 NextBlockAddr = alignToBlock(NextBlockAddr, *B);
248 B->setAddress(NextBlockAddr);
249 NextBlockAddr += B->getSize();
325250 }
326251 }
327252
329254 }
330255
331256 DenseSet JITLinkerBase::getExternalSymbolNames() const {
332 // Identify unresolved external atoms.
257 // Identify unresolved external symbols.
333258 DenseSet UnresolvedExternals;
334 for (auto *DA : G->external_atoms()) {
335 assert(DA->getAddress() == 0 &&
259 for (auto *Sym : G->external_symbols()) {
260 assert(Sym->getAddress() == 0 &&
336261 "External has already been assigned an address");
337 assert(DA->getName() != StringRef() && DA->getName() != "" &&
262 assert(Sym->getName() != StringRef() && Sym->getName() != "" &&
338263 "Externals must be named");
339 UnresolvedExternals.insert(DA->getName());
264 UnresolvedExternals.insert(Sym->getName());
340265 }
341266 return UnresolvedExternals;
342267 }
343268
344269 void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
345 for (auto &KV : Result) {
346 Atom &A = G->getAtomByName(KV.first);
347 assert(A.getAddress() == 0 && "Atom already resolved");
348 A.setAddress(KV.second.getAddress());
270 for (auto *Sym : G->external_symbols()) {
271 assert(Sym->getAddress() == 0 && "Symbol already resolved");
272 assert(!Sym->isDefined() && "Symbol being resolved is already defined");
273 assert(Result.count(Sym->getName()) && "Missing resolution for symbol");
274 Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress());
349275 }
350276
351277 LLVM_DEBUG({
352278 dbgs() << "Externals after applying lookup result:\n";
353 for (auto *A : G->external_atoms())
354 dbgs() << " " << A->getName() << ": "
355 << formatv("{0:x16}", A->getAddress()) << "\n";
356 });
357 assert(llvm::all_of(G->external_atoms(),
358 [](Atom *A) { return A->getAddress() != 0; }) &&
359 "All atoms should have been resolved by this point");
279 for (auto *Sym : G->external_symbols())
280 dbgs() << " " << Sym->getName() << ": "
281 << formatv("{0:x16}", Sym->getAddress()) << "\n";
282 });
283 assert(llvm::all_of(G->external_symbols(),
284 [](Symbol *Sym) { return Sym->getAddress() != 0; }) &&
285 "All symbols should have been resolved by this point");
360286 }
361287
362288 void JITLinkerBase::deallocateAndBailOut(Error Err) {
370296 G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); });
371297 }
372298
373 void prune(AtomGraph &G) {
374 std::vector Worklist;
375 DenseMap> EdgesToUpdate;
376
377 // Build the initial worklist from all atoms initially live.
378 for (auto *DA : G.defined_atoms()) {
379 if (!DA->isLive() || DA->shouldDiscard())
299 void prune(LinkGraph &G) {
300 std::vector Worklist;
301 DenseSet VisitedBlocks;
302
303 // Build the initial worklist from all symbols initially live.
304 for (auto *Sym : G.defined_symbols())
305 if (Sym->isLive())
306 Worklist.push_back(Sym);
307
308 // Propagate live flags to all symbols reachable from the initial live set.
309 while (!Worklist.empty()) {
310 auto *Sym = Worklist.back();
311 Worklist.pop_back();
312
313 auto &B = Sym->getBlock();
314
315 // Skip addressables that we've visited before.
316 if (VisitedBlocks.count(&B))
380317 continue;
381318
382 for (auto &E : DA->edges()) {
383 if (!E.getTarget().isDefined())
384 continue;
385
386 auto &EDT = static_cast(E.getTarget());
387
388 if (EDT.shouldDiscard())
389 EdgesToUpdate[&EDT].push_back(&E);
390 else if (E.isKeepAlive() && !EDT.isLive())
391 Worklist.push_back(&EDT);
392 }
393 }
394
395 // Propagate live flags to all atoms reachable from the initial live set.
396 while (!Worklist.empty()) {
397 DefinedAtom &NextLive = *Worklist.back();
398 Worklist.pop_back();
399
400 assert(!NextLive.shouldDiscard() &&
401 "should-discard nodes should never make it into the worklist");
402
403 // If this atom has already been marked as live, or is marked to be
404 // discarded, then skip it.
405 if (NextLive.isLive())
406 continue;
407
408 // Otherwise set it as live and add any non-live atoms that it points to
409 // to the worklist.
410 NextLive.setLive(true);
411
412 for (auto &E : NextLive.edges()) {
413 if (!E.getTarget().isDefined())
414 continue;
415
416 auto &EDT = static_cast(E.getTarget());
417
418 if (EDT.shouldDiscard())
419 EdgesToUpdate[&EDT].push_back(&E);
420 else if (E.isKeepAlive() && !EDT.isLive())
421 Worklist.push_back(&EDT);
422 }
423 }
424
425 // Collect atoms to remove, then remove them from the graph.
426 std::vector AtomsToRemove;
427 for (auto *DA : G.defined_atoms())
428 if (DA->shouldDiscard() || !DA->isLive())
429 AtomsToRemove.push_back(DA);
430
431 LLVM_DEBUG(dbgs() << "Pruning atoms:\n");
432 for (auto *DA : AtomsToRemove) {
433 LLVM_DEBUG(dbgs() << " " << *DA << "... ");
434
435 // Check whether we need to replace this atom with an external atom.
436 //
437 // We replace if all of the following hold:
438 // (1) The atom is marked should-discard,
439 // (2) it has live edges (i.e. edges from live atoms) pointing to it.
440 //
441 // Otherwise we simply delete the atom.
442
443 G.removeDefinedAtom(*DA);
444
445 auto EdgesToUpdateItr = EdgesToUpdate.find(DA);
446 if (EdgesToUpdateItr != EdgesToUpdate.end()) {
447 auto &ExternalReplacement = G.addExternalAtom(DA->getName());
448 for (auto *EdgeToUpdate : EdgesToUpdateItr->second)
449 EdgeToUpdate->setTarget(ExternalReplacement);
450 LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n");
451 } else
452 LLVM_DEBUG(dbgs() << "deleted\n");
453 }
454
455 // Finally, discard any absolute symbols that were marked should-discard.
319 VisitedBlocks.insert(&B);
320
321 for (auto &E : Sym->getBlock().edges()) {
322 if (E.getTarget().isDefined() && !E.getTarget().isLive()) {
323 E.getTarget().setLive(true);
324 Worklist.push_back(&E.getTarget());
325 }
326 }
327 }
328
329 // Collect all the symbols to remove, then remove them.
456330 {
457 std::vector AbsoluteAtomsToRemove;
458 for (auto *A : G.absolute_atoms())
459 if (A->shouldDiscard() || A->isLive())
460 AbsoluteAtomsToRemove.push_back(A);
461 for (auto *A : AbsoluteAtomsToRemove)
462 G.removeAbsoluteAtom(*A);
331 LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n");
332 std::vector SymbolsToRemove;
333 for (auto *Sym : G.defined_symbols())
334 if (!Sym->isLive())
335 SymbolsToRemove.push_back(Sym);
336 for (auto *Sym : SymbolsToRemove) {
337 LLVM_DEBUG(dbgs() << " " << *Sym << "...\n");
338 G.removeDefinedSymbol(*Sym);
339 }
340 }
341
342 // Delete any unused blocks.
343 {
344 LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n");
345 std::vector BlocksToRemove;
346 for (auto *B : G.blocks())
347 if (!VisitedBlocks.count(B))
348 BlocksToRemove.push_back(B);
349 for (auto *B : BlocksToRemove) {
350 LLVM_DEBUG(dbgs() << " " << *B << "...\n");
351 G.removeBlock(*B);
352 }
463353 }
464354 }
465355
4040
4141 protected:
4242 struct SegmentLayout {
43 using SectionAtomsList = std::vector;
44 struct SectionLayout {
45 SectionLayout(Section &S) : S(&S) {}
46
47 Section *S;
48 SectionAtomsList Atoms;
49 };
50
51 using SectionLayoutList = std::vector;
52
53 SectionLayoutList ContentSections;
54 SectionLayoutList ZeroFillSections;
43 using BlocksList = std::vector;
44
45 BlocksList ContentBlocks;
46 BlocksList ZeroFillBlocks;
5547 };
5648
5749 using SegmentLayoutMap = DenseMap;
5850
5951 // Phase 1:
60 // 1.1: Build atom graph
52 // 1.1: Build link graph
6153 // 1.2: Run pre-prune passes
6254 // 1.2: Prune graph
6355 // 1.3: Run post-prune passes
64 // 1.4: Sort atoms into segments
56 // 1.4: Sort blocks into segments
6557 // 1.5: Allocate segment memory
6658 // 1.6: Identify externals and make an async call to resolve function
6759 void linkPhase1(std::unique_ptr Self);
6860
6961 // Phase 2:
7062 // 2.1: Apply resolution results
71 // 2.2: Fix up atom contents
63 // 2.2: Fix up block contents
7264 // 2.3: Call OnResolved callback
7365 // 2.3: Make an async call to transfer and finalize memory.
7466 void linkPhase2(std::unique_ptr Self,
75 Expected LookupResult);
67 Expected LookupResult,
68 SegmentLayoutMap Layout);
7669
7770 // Phase 3:
7871 // 3.1: Call OnFinalized callback, handing off allocation.
8073
8174 // Build a graph from the given object buffer.
8275 // To be implemented by the client.
83 virtual ExpectedAtomGraph>>
76 virtual ExpectedLinkGraph>>
8477 buildGraph(MemoryBufferRef ObjBuffer) = 0;
8578
86 // For debug dumping of the atom graph.
79 // For debug dumping of the link graph.
8780 virtual StringRef getEdgeKindName(Edge::Kind K) const = 0;
81
82 // Alight a JITTargetAddress to conform with block alignment requirements.
83 static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) {
84 uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment();
85 return Addr + Delta;
86 }
87
88 // Alight a pointer to conform with block alignment requirements.
89 static char *alignToBlock(char *P, Block &B) {
90 uint64_t PAddr = static_cast(reinterpret_cast(P));
91 uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment();
92 return P + Delta;
93 }
8894
8995 private:
9096 // Run all passes in the given pass list, bailing out immediately if any pass
9197 // returns an error.
92 Error runPasses(AtomGraphPassList &Passes, AtomGraph &G);
93
94 // Copy atom contents and apply relocations.
98 Error runPasses(LinkGraphPassList &Passes);
99
100 // Copy block contents and apply relocations.
95101 // Implemented in JITLinker.
96102 virtual Error
97 copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout,
98 JITLinkMemoryManager::Allocation &Alloc) const = 0;
99
100 void layOutAtoms();
103 copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
104 JITLinkMemoryManager::Allocation &Alloc) const = 0;
105
106 SegmentLayoutMap layOutBlocks();
101107 Error allocateSegments(const SegmentLayoutMap &Layout);
102108 DenseSet getExternalSymbolNames() const;
103109 void applyLookupResult(AsyncLookupResult LR);
107113
108114 std::unique_ptr Ctx;
109115 PassConfiguration Passes;
110 std::unique_ptr G;
111 SegmentLayoutMap Layout;
116 std::unique_ptr G;
112117 std::unique_ptr Alloc;
113118 };
114119
139144 }
140145
141146 Error
142 copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout,
143 JITLinkMemoryManager::Allocation &Alloc) const override {
144 LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n");
147 copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
148 JITLinkMemoryManager::Allocation &Alloc) const override {
149 LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n");
145150 for (auto &KV : Layout) {
146151 auto &Prot = KV.first;
147152 auto &SegLayout = KV.second;
148153
149154 auto SegMem = Alloc.getWorkingMemory(
150155 static_cast(Prot));
151 char *LastAtomEnd = SegMem.data();
152 char *AtomDataPtr = LastAtomEnd;
156 char *LastBlockEnd = SegMem.data();
157 char *BlockDataPtr = LastBlockEnd;
153158
154159 LLVM_DEBUG({
155160 dbgs() << " Processing segment "
159164 << " ]\n Processing content sections:\n";
160165 });
161166
162 for (auto &SI : SegLayout.ContentSections) {
163 LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n");
164
165 AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment());
167 for (auto *B : SegLayout.ContentBlocks) {
168 LLVM_DEBUG(dbgs() << " " << *B << ":\n");
169
170 // Pad to alignment/alignment-offset.
171 BlockDataPtr = alignToBlock(BlockDataPtr, *B);
166172
167173 LLVM_DEBUG({
168 dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr
169 << " to meet section alignment "
170 << " of " << SI.S->getAlignment() << "\n";
174 dbgs() << " Bumped block pointer to "
175 << (const void *)BlockDataPtr << " to meet block alignment "
176 << B->getAlignment() << " and alignment offset "
177 << B->getAlignmentOffset() << "\n";
171178 });
172179
173 for (auto *DA : SI.Atoms) {
174
175 // Align.
176 AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment());
177 LLVM_DEBUG({
178 dbgs() << " Bumped atom pointer to "
179 << (const void *)AtomDataPtr << " to meet alignment of "
180 << DA->getAlignment() << "\n";
181 });
182
183 // Zero pad up to alignment.
184 LLVM_DEBUG({
185 if (LastAtomEnd != AtomDataPtr)
186 dbgs() << " Zero padding from " << (const void *)LastAtomEnd
187 << " to " << (const void *)AtomDataPtr << "\n";
188 });
189 while (LastAtomEnd != AtomDataPtr)
190 *LastAtomEnd++ = 0;
191
192 // Copy initial atom content.
193 LLVM_DEBUG({
194 dbgs() << " Copying atom " << *DA << " content, "
195 << DA->getContent().size() << " bytes, from "
196 << (const void *)DA->getContent().data() << " to "
197 << (const void *)AtomDataPtr << "\n";
198 });
199 memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size());
200
201 // Copy atom data and apply fixups.
202 LLVM_DEBUG(dbgs() << " Applying fixups.\n");
203 for (auto &E : DA->edges()) {
204
205 // Skip non-relocation edges.
206 if (!E.isRelocation())
207 continue;
208
209 // Dispatch to LinkerImpl for fixup.
210 if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr))
211 return Err;
212 }
213
214 // Point the atom's content to the fixed up buffer.
215 DA->setContent(StringRef(AtomDataPtr, DA->getContent().size()));
216
217 // Update atom end pointer.
218 LastAtomEnd = AtomDataPtr + DA->getContent().size();
219 AtomDataPtr = LastAtomEnd;
180 // Zero pad up to alignment.
181 LLVM_DEBUG({
182 if (LastBlockEnd != BlockDataPtr)
183 dbgs() << " Zero padding from " << (const void *)LastBlockEnd
184 << " to " << (const void *)BlockDataPtr << "\n";
185 });
186
187 while (LastBlockEnd != BlockDataPtr)
188 *LastBlockEnd++ = 0;
189
190 // Copy initial block content.
191 LLVM_DEBUG({
192 dbgs() << " Copying block " << *B << " content, "
193 << B->getContent().size() << " bytes, from "
194 << (const void *)B->getContent().data() << " to "
195 << (const void *)BlockDataPtr << "\n";
196 });
197 memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size());
198
199 // Copy Block data and apply fixups.
200 LLVM_DEBUG(dbgs() << " Applying fixups.\n");
201 for (auto &E : B->edges()) {
202
203 // Skip non-relocation edges.
204 if (!E.isRelocation())
205 continue;
206
207 // Dispatch to LinkerImpl for fixup.
208 if (auto Err = impl().applyFixup(*B, E, BlockDataPtr))
209 return Err;
220210 }
211
212 // Point the block's content to the fixed up buffer.
213 B->setContent(StringRef(BlockDataPtr, B->getContent().size()));
214
215 // Update block end pointer.
216 LastBlockEnd = BlockDataPtr + B->getContent().size();
217 BlockDataPtr = LastBlockEnd;
221218 }
222219
223220 // Zero pad the rest of the segment.
224221 LLVM_DEBUG({
225222 dbgs() << " Zero padding end of segment from "
226 << (const void *)LastAtomEnd << " to "
223 << (const void *)LastBlockEnd << " to "
227224 << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n";
228225 });
229 while (LastAtomEnd != SegMem.data() + SegMem.size())
230 *LastAtomEnd++ = 0;
226 while (LastBlockEnd != SegMem.data() + SegMem.size())
227 *LastBlockEnd++ = 0;
231228 }
232229
233230 return Error::success();
234231 }
235232 };
236233
237 /// Dead strips and replaces discarded definitions with external atoms.
234 /// Removes dead symbols/blocks/addressables.
238235 ///
239 /// Finds the set of nodes reachable from any node initially marked live
240 /// (nodes marked should-discard are treated as not live, even if they are
241 /// reachable). All nodes not marked as live at the end of this process,
242 /// are deleted. Nodes that are live, but marked should-discard are replaced
243 /// with external atoms and all edges to them are re-written.
244 void prune(AtomGraph &G);
245
246 Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
247 StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
248 Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind);
236 /// Finds the set of symbols and addressables reachable from any symbol
237 /// initially marked live. All symbols/addressables not marked live at the end
238 /// of this process are removed.
239 void prune(LinkGraph &G);
249240
250241 } // end namespace jitlink
251242 } // end namespace llvm
6060 AllocationMap SegBlocks;
6161 };
6262
63 if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate()))
64 return make_error("Page size is not a power of 2",
65 inconvertibleErrorCode());
66
6367 AllocationMap Blocks;
6468 const sys::Memory::ProtectionFlags ReadWrite =
6569 static_cast(sys::Memory::MF_READ |
6872 for (auto &KV : Request) {
6973 auto &Seg = KV.second;
7074
71 if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate())
75 if (Seg.getAlignment() > sys::Process::getPageSizeEstimate())
7276 return make_error("Cannot request higher than page "
7377 "alignment",
7478 inconvertibleErrorCode());
7579
76 if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0)
77 return make_error("Page size is not a multiple of "
78 "alignment",
79 inconvertibleErrorCode());
80
81 uint64_t ZeroFillStart =
82 alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment());
83 uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize();
80 uint64_t SegmentSize = Seg.getContentSize() + Seg.getZeroFillSize();
8481
8582 std::error_code EC;
8683 auto SegMem =
9087 return errorCodeToError(EC);
9188
9289 // Zero out the zero-fill memory.
93 memset(static_cast(SegMem.base()) + ZeroFillStart, 0,
90 memset(static_cast(SegMem.base()) + Seg.getContentSize(), 0,
9491 Seg.getZeroFillSize());
9592
9693 // Record the block for this segment.
+0
-412
lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp less more
None //=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===//
1 //
2 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3 // See https://llvm.org/LICENSE.txt for license information.
4 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // Generic MachO AtomGraph buliding code.
9 //
10 //===----------------------------------------------------------------------===//
11
12 #include "MachOAtomGraphBuilder.h"
13
14 #define DEBUG_TYPE "jitlink"
15
16 namespace llvm {
17 namespace jitlink {
18
19 MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {}
20
21 Expected> MachOAtomGraphBuilder::buildGraph() {
22 if (auto Err = parseSections())
23 return std::move(Err);
24
25 if (auto Err = addAtoms())
26 return std::move(Err);
27
28 if (auto Err = addRelocations())
29 return std::move(Err);
30
31 return std::move(G);
32 }
33
34 MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj)
35 : Obj(Obj),
36 G(std::make_unique(Obj.getFileName(), getPointerSize(Obj),
37 getEndianness(Obj))) {}
38
39 void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName,
40 CustomAtomizeFunction Atomizer) {
41 assert(!CustomAtomizeFunctions.count(SectionName) &&
42 "Custom atomizer for this section already exists");
43 CustomAtomizeFunctions[SectionName] = std::move(Atomizer);
44 }
45
46 bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) {
47 // If these atoms are the same then they're trivially "locked".
48 if (&A == &B)
49 return true;
50
51 // If A and B are different, check whether either is undefined. (in which
52 // case they are not locked).
53 if (!A.isDefined() || !B.isDefined())
54 return false;
55
56 // A and B are different, but they're both defined atoms. We need to check
57 // whether they're part of the same alt_entry chain.
58 auto &DA = static_cast(A);
59 auto &DB = static_cast(B);
60
61 auto AStartItr = AltEntryStarts.find(&DA);
62 if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out.
63 return false;
64
65 auto BStartItr = AltEntryStarts.find(&DB);
66 if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out.
67 return false;
68
69 // A and B are layout locked if they're in the same chain.
70 return AStartItr->second == BStartItr->second;
71 }
72
73 unsigned
74 MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) {
75 return Obj.is64Bit() ? 8 : 4;
76 }
77
78 support::endianness
79 MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) {
80 return Obj.isLittleEndian() ? support::little : support::big;
81 }
82
83 MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() {
84 if (!CommonSymbolsSection) {
85 auto Prot = static_cast(
86 sys::Memory::MF_READ | sys::Memory::MF_WRITE);
87 auto &GenericSection = G->createSection("", 1, Prot, true);
88 CommonSymbolsSection = MachOSection(GenericSection);
89 }
90 return *CommonSymbolsSection;
91 }
92
93 Error MachOAtomGraphBuilder::parseSections() {
94 for (auto &SecRef : Obj.sections()) {
95 assert((SecRef.getAlignment() <= std::numeric_limits::max()) &&
96 "Section alignment does not fit in 32 bits");
97
98 Expected NameOrErr = SecRef.getName();
99 if (!NameOrErr)
100 return NameOrErr.takeError();
101 StringRef Name = *NameOrErr;
102
103 unsigned SectionIndex = SecRef.getIndex() + 1;
104
105 uint32_t Align = SecRef.getAlignment();
106 if (!isPowerOf2_32(Align))
107 return make_error("Section " + Name +
108 " has non-power-of-2 "
109 "alignment");
110
111 // FIXME: Get real section permissions
112 // How, exactly, on MachO?
113 sys::Memory::ProtectionFlags Prot;
114 if (SecRef.isText())
115 Prot = static_cast(sys::Memory::MF_READ |
116 sys::Memory::MF_EXEC);
117 else
118 Prot = static_cast(sys::Memory::MF_READ |
119 sys::Memory::MF_WRITE);
120
121 auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS());
122
123 LLVM_DEBUG({
124 dbgs() << "Adding section " << Name << ": "
125 << format("0x%016" PRIx64, SecRef.getAddress())
126 << ", align: " << SecRef.getAlignment() << "\n";
127 });
128
129 assert(!Sections.count(SectionIndex) && "Section index already in use");
130
131 auto &MachOSec =
132 Sections
133 .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(),
134 SecRef.getAlignment())
135 .first->second;
136
137 if (!SecRef.isVirtual()) {
138 // If this section has content then record it.
139 Expected Content = SecRef.getContents();
140 if (!Content)
141 return Content.takeError();
142 if (Content->size() != SecRef.getSize())
143 return make_error("Section content size does not match "
144 "declared size for " +
145 Name);
146 MachOSec.setContent(*Content);
147 } else {
148 // If this is a zero-fill section then just record the size.
149 MachOSec.setZeroFill(SecRef.getSize());
150 }
151
152 uint32_t SectionFlags =
153 Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags