llvm.org GIT mirror llvm / 8907b4b
split the JIT memory management code out from the main JIT logic into its own JITMemoryManager interface. There is no functionality change with this patch. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@44640 91177308-0d34-0410-b5e6-96231b3b80d8 Chris Lattner 12 years ago
4 changed file(s) with 552 addition(s) and 413 deletion(s). Raw diff Collapse all Expand all
1111 //
1212 //===----------------------------------------------------------------------===//
1313
14 #ifndef EXECUTION_ENGINE_JIT_H
15 #define EXECUTION_ENGINE_JIT_H
14 #ifndef LLVM_EXECUTION_ENGINE_JIT_H
15 #define LLVM_EXECUTION_ENGINE_JIT_H
1616
1717 #include "llvm/ExecutionEngine/ExecutionEngine.h"
1818 #include
0 //===-- JITMemoryManager.h - Interface JIT uses to Allocate Mem -*- C++ -*-===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file was developed by Chris Lattner and is distributed under the
5 // University of Illinois Open Source License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the JITMemoryManagerInterface
10 //
11 //===----------------------------------------------------------------------===//
12
13 #ifndef LLVM_EXECUTION_ENGINE_JIT_H
14 #define LLVM_EXECUTION_ENGINE_JIT_H
15
16 #include "llvm/Support/DataTypes.h"
17
18 namespace llvm {
19 class Function;
20
21 /// JITMemoryManager - This interface is used by the JIT to allocate and manage
22 /// memory for the code generated by the JIT. This can be reimplemented by
23 /// clients that have a strong desire to control how the layout of JIT'd memory
24 /// works.
25 class JITMemoryManager {
26 protected:
27 bool HasGOT;
28 public:
29 JITMemoryManager() : HasGOT(false) {}
30 virtual ~JITMemoryManager();
31
32 /// CreateDefaultMemManager - This is used to create the default
33 /// JIT Memory Manager if the client does not provide one to the JIT.
34 static JITMemoryManager *CreateDefaultMemManager();
35
36 //===--------------------------------------------------------------------===//
37 // Global Offset Table Management
38 //===--------------------------------------------------------------------===//
39
40 /// AllocateGOT - If the current table requires a Global Offset Table, this
41 /// method is invoked to allocate it. This method is required to set HasGOT
42 /// to true.
43 virtual void AllocateGOT() = 0;
44
45 /// isManagingGOT - Return true if the AllocateGOT method is called.
46 ///
47 bool isManagingGOT() const {
48 return HasGOT;
49 }
50
51 /// getGOTBase - If this is managing a Global Offset Table, this method should
52 /// return a pointer to its base.
53 virtual unsigned char *getGOTBase() const = 0;
54
55 //===--------------------------------------------------------------------===//
56 // Main Allocation Functions
57 //===--------------------------------------------------------------------===//
58
59 /// startFunctionBody - When we start JITing a function, the JIT calls this
60 /// method to allocate a block of free RWX memory, which returns a pointer to
61 /// it. The JIT doesn't know ahead of time how much space it will need to
62 /// emit the function, so it doesn't pass in the size. Instead, this method
63 /// is required to pass back a "valid size". The JIT will be careful to not
64 /// write more than the returned ActualSize bytes of memory.
65 virtual unsigned char *startFunctionBody(const Function *F,
66 uintptr_t &ActualSize) = 0;
67
68 /// allocateStub - This method is called by the JIT to allocate space for a
69 /// function stub (used to handle limited branch displacements) while it is
70 /// JIT compiling a function. For example, if foo calls bar, and if bar
71 /// either needs to be lazily compiled or is a native function that exists too
72 /// far away from the call site to work, this method will be used to make a
73 /// thunk for it. The stub should be "close" to the current function body,
74 /// but should not be included in the 'actualsize' returned by
75 /// startFunctionBody.
76 virtual unsigned char *allocateStub(unsigned StubSize, unsigned Alignment) =0;
77
78
79 /// endFunctionBody - This method is called when the JIT is done codegen'ing
80 /// the specified function. At this point we know the size of the JIT
81 /// compiled function. This passes in FunctionStart (which was returned by
82 /// the startFunctionBody method) and FunctionEnd which is a pointer to the
83 /// actual end of the function. This method should mark the space allocated
84 /// and remember where it is in case the client wants to deallocate it.
85 virtual void endFunctionBody(const Function *F, unsigned char *FunctionStart,
86 unsigned char *FunctionEnd) = 0;
87
88 /// deallocateMemForFunction - Free JIT memory for the specified function.
89 /// This is never called when the JIT is currently emitting a function.
90 virtual void deallocateMemForFunction(const Function *F) = 0;
91 };
92
93 } // end namespace llvm.
94
95 #endif
2121 #include "llvm/CodeGen/MachineConstantPool.h"
2222 #include "llvm/CodeGen/MachineJumpTableInfo.h"
2323 #include "llvm/CodeGen/MachineRelocation.h"
24 #include "llvm/ExecutionEngine/GenericValue.h"
24 #include "llvm/ExecutionEngine/JITMemoryManager.h"
2525 #include "llvm/Target/TargetData.h"
2626 #include "llvm/Target/TargetJITInfo.h"
2727 #include "llvm/Target/TargetMachine.h"
2929 #include "llvm/Support/MutexGuard.h"
3030 #include "llvm/System/Disassembler.h"
3131 #include "llvm/ADT/Statistic.h"
32 #include "llvm/System/Memory.h"
3332 #include
3433 using namespace llvm;
3534
3736 STATISTIC(NumRelos, "Number of relocations applied");
3837 static JIT *TheJIT = 0;
3938
40 //===----------------------------------------------------------------------===//
41 // JITMemoryManager code.
42 //
43 namespace {
44 /// MemoryRangeHeader - For a range of memory, this is the header that we put
45 /// on the block of memory. It is carefully crafted to be one word of memory.
46 /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
47 /// which starts with this.
48 struct FreeRangeHeader;
49 struct MemoryRangeHeader {
50 /// ThisAllocated - This is true if this block is currently allocated. If
51 /// not, this can be converted to a FreeRangeHeader.
52 unsigned ThisAllocated : 1;
53
54 /// PrevAllocated - Keep track of whether the block immediately before us is
55 /// allocated. If not, the word immediately before this header is the size
56 /// of the previous block.
57 unsigned PrevAllocated : 1;
58
59 /// BlockSize - This is the size in bytes of this memory block,
60 /// including this header.
61 uintptr_t BlockSize : (sizeof(intptr_t)*8 - 2);
62
63
64 /// getBlockAfter - Return the memory block immediately after this one.
65 ///
66 MemoryRangeHeader &getBlockAfter() const {
67 return *(MemoryRangeHeader*)((char*)this+BlockSize);
68 }
69
70 /// getFreeBlockBefore - If the block before this one is free, return it,
71 /// otherwise return null.
72 FreeRangeHeader *getFreeBlockBefore() const {
73 if (PrevAllocated) return 0;
74 intptr_t PrevSize = ((intptr_t *)this)[-1];
75 return (FreeRangeHeader*)((char*)this-PrevSize);
76 }
77
78 /// FreeBlock - Turn an allocated block into a free block, adjusting
79 /// bits in the object headers, and adding an end of region memory block.
80 FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
81
82 /// TrimAllocationToSize - If this allocated block is significantly larger
83 /// than NewSize, split it into two pieces (where the former is NewSize
84 /// bytes, including the header), and add the new block to the free list.
85 FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
86 uint64_t NewSize);
87 };
88
89 /// FreeRangeHeader - For a memory block that isn't already allocated, this
90 /// keeps track of the current block and has a pointer to the next free block.
91 /// Free blocks are kept on a circularly linked list.
92 struct FreeRangeHeader : public MemoryRangeHeader {
93 FreeRangeHeader *Prev;
94 FreeRangeHeader *Next;
95
96 /// getMinBlockSize - Get the minimum size for a memory block. Blocks
97 /// smaller than this size cannot be created.
98 static unsigned getMinBlockSize() {
99 return sizeof(FreeRangeHeader)+sizeof(intptr_t);
100 }
101
102 /// SetEndOfBlockSizeMarker - The word at the end of every free block is
103 /// known to be the size of the free block. Set it for this block.
104 void SetEndOfBlockSizeMarker() {
105 void *EndOfBlock = (char*)this + BlockSize;
106 ((intptr_t *)EndOfBlock)[-1] = BlockSize;
107 }
108
109 FreeRangeHeader *RemoveFromFreeList() {
110 assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
111 Next->Prev = Prev;
112 return Prev->Next = Next;
113 }
114
115 void AddToFreeList(FreeRangeHeader *FreeList) {
116 Next = FreeList;
117 Prev = FreeList->Prev;
118 Prev->Next = this;
119 Next->Prev = this;
120 }
121
122 /// GrowBlock - The block after this block just got deallocated. Merge it
123 /// into the current block.
124 void GrowBlock(uintptr_t NewSize);
125
126 /// AllocateBlock - Mark this entire block allocated, updating freelists
127 /// etc. This returns a pointer to the circular free-list.
128 FreeRangeHeader *AllocateBlock();
129 };
130 }
131
132
133 /// AllocateBlock - Mark this entire block allocated, updating freelists
134 /// etc. This returns a pointer to the circular free-list.
135 FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
136 assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
137 "Cannot allocate an allocated block!");
138 // Mark this block allocated.
139 ThisAllocated = 1;
140 getBlockAfter().PrevAllocated = 1;
141
142 // Remove it from the free list.
143 return RemoveFromFreeList();
144 }
145
146 /// FreeBlock - Turn an allocated block into a free block, adjusting
147 /// bits in the object headers, and adding an end of region memory block.
148 /// If possible, coalesce this block with neighboring blocks. Return the
149 /// FreeRangeHeader to allocate from.
150 FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
151 MemoryRangeHeader *FollowingBlock = &getBlockAfter();
152 assert(ThisAllocated && "This block is already allocated!");
153 assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
154
155 FreeRangeHeader *FreeListToReturn = FreeList;
156
157 // If the block after this one is free, merge it into this block.
158 if (!FollowingBlock->ThisAllocated) {
159 FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
160 // "FreeList" always needs to be a valid free block. If we're about to
161 // coalesce with it, update our notion of what the free list is.
162 if (&FollowingFreeBlock == FreeList) {
163 FreeList = FollowingFreeBlock.Next;
164 FreeListToReturn = 0;
165 assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
166 }
167 FollowingFreeBlock.RemoveFromFreeList();
168
169 // Include the following block into this one.
170 BlockSize += FollowingFreeBlock.BlockSize;
171 FollowingBlock = &FollowingFreeBlock.getBlockAfter();
172
173 // Tell the block after the block we are coalescing that this block is
174 // allocated.
175 FollowingBlock->PrevAllocated = 1;
176 }
177
178 assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
179
180 if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
181 PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
182 return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
183 }
184
185 // Otherwise, mark this block free.
186 FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
187 FollowingBlock->PrevAllocated = 0;
188 FreeBlock.ThisAllocated = 0;
189
190 // Link this into the linked list of free blocks.
191 FreeBlock.AddToFreeList(FreeList);
192
193 // Add a marker at the end of the block, indicating the size of this free
194 // block.
195 FreeBlock.SetEndOfBlockSizeMarker();
196 return FreeListToReturn ? FreeListToReturn : &FreeBlock;
197 }
198
199 /// GrowBlock - The block after this block just got deallocated. Merge it
200 /// into the current block.
201 void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
202 assert(NewSize > BlockSize && "Not growing block?");
203 BlockSize = NewSize;
204 SetEndOfBlockSizeMarker();
205 getBlockAfter().PrevAllocated = 0;
206 }
207
208 /// TrimAllocationToSize - If this allocated block is significantly larger
209 /// than NewSize, split it into two pieces (where the former is NewSize
210 /// bytes, including the header), and add the new block to the free list.
211 FreeRangeHeader *MemoryRangeHeader::
212 TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
213 assert(ThisAllocated && getBlockAfter().PrevAllocated &&
214 "Cannot deallocate part of an allocated block!");
215
216 // Round up size for alignment of header.
217 unsigned HeaderAlign = __alignof(FreeRangeHeader);
218 NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
219
220 // Size is now the size of the block we will remove from the start of the
221 // current block.
222 assert(NewSize <= BlockSize &&
223 "Allocating more space from this block than exists!");
224
225 // If splitting this block will cause the remainder to be too small, do not
226 // split the block.
227 if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
228 return FreeList;
229
230 // Otherwise, we splice the required number of bytes out of this block, form
231 // a new block immediately after it, then mark this block allocated.
232 MemoryRangeHeader &FormerNextBlock = getBlockAfter();
233
234 // Change the size of this block.
235 BlockSize = NewSize;
236
237 // Get the new block we just sliced out and turn it into a free block.
238 FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
239 NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
240 NewNextBlock.ThisAllocated = 0;
241 NewNextBlock.PrevAllocated = 1;
242 NewNextBlock.SetEndOfBlockSizeMarker();
243 FormerNextBlock.PrevAllocated = 0;
244 NewNextBlock.AddToFreeList(FreeList);
245 return &NewNextBlock;
246 }
247
248
249 namespace {
250 /// JITMemoryManager - Manage memory for the JIT code generation in a logical,
251 /// sane way. This splits a large block of MAP_NORESERVE'd memory into two
252 /// sections, one for function stubs, one for the functions themselves. We
253 /// have to do this because we may need to emit a function stub while in the
254 /// middle of emitting a function, and we don't know how large the function we
255 /// are emitting is. This never bothers to release the memory, because when
256 /// we are ready to destroy the JIT, the program exits.
257 class JITMemoryManager {
258 std::vector Blocks; // Memory blocks allocated by the JIT
259 FreeRangeHeader *FreeMemoryList; // Circular list of free blocks.
260
261 // When emitting code into a memory block, this is the block.
262 MemoryRangeHeader *CurBlock;
263
264 unsigned char *CurStubPtr, *StubBase;
265 unsigned char *GOTBase; // Target Specific reserved memory
266
267 // Centralize memory block allocation.
268 sys::MemoryBlock getNewMemoryBlock(unsigned size);
269
270 std::map FunctionBlocks;
271 public:
272 JITMemoryManager(bool useGOT);
273 ~JITMemoryManager();
274
275 inline unsigned char *allocateStub(unsigned StubSize, unsigned Alignment);
276
277 /// startFunctionBody - When a function starts, allocate a block of free
278 /// executable memory, returning a pointer to it and its actual size.
279 unsigned char *startFunctionBody(uintptr_t &ActualSize) {
280 CurBlock = FreeMemoryList;
281
282 // Allocate the entire memory block.
283 FreeMemoryList = FreeMemoryList->AllocateBlock();
284 ActualSize = CurBlock->BlockSize-sizeof(MemoryRangeHeader);
285 return (unsigned char *)(CurBlock+1);
286 }
287
288 /// endFunctionBody - The function F is now allocated, and takes the memory
289 /// in the range [FunctionStart,FunctionEnd).
290 void endFunctionBody(const Function *F, unsigned char *FunctionStart,
291 unsigned char *FunctionEnd) {
292 assert(FunctionEnd > FunctionStart);
293 assert(FunctionStart == (unsigned char *)(CurBlock+1) &&
294 "Mismatched function start/end!");
295
296 uintptr_t BlockSize = FunctionEnd - (unsigned char *)CurBlock;
297 FunctionBlocks[F] = CurBlock;
298
299 // Release the memory at the end of this block that isn't needed.
300 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
301 }
302
303 unsigned char *getGOTBase() const {
304 return GOTBase;
305 }
306 bool isManagingGOT() const {
307 return GOTBase != NULL;
308 }
309
310 /// deallocateMemForFunction - Deallocate all memory for the specified
311 /// function body.
312 void deallocateMemForFunction(const Function *F) {
313 std::map::iterator
314 I = FunctionBlocks.find(F);
315 if (I == FunctionBlocks.end()) return;
316
317 // Find the block that is allocated for this function.
318 MemoryRangeHeader *MemRange = I->second;
319 assert(MemRange->ThisAllocated && "Block isn't allocated!");
320
321 // Fill the buffer with garbage!
322 DEBUG(memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)));
323
324 // Free the memory.
325 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
326
327 // Finally, remove this entry from FunctionBlocks.
328 FunctionBlocks.erase(I);
329 }
330 };
331 }
332
333 JITMemoryManager::JITMemoryManager(bool useGOT) {
334 // Allocate a 16M block of memory for functions.
335 sys::MemoryBlock MemBlock = getNewMemoryBlock(16 << 20);
336
337 unsigned char *MemBase = reinterpret_cast(MemBlock.base());
338
339 // Allocate stubs backwards from the base, allocate functions forward
340 // from the base.
341 StubBase = MemBase;
342 CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards.
343
344 // We set up the memory chunk with 4 mem regions, like this:
345 // [ START
346 // [ Free #0 ] -> Large space to allocate functions from.
347 // [ Allocated #1 ] -> Tiny space to separate regions.
348 // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
349 // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
350 // END ]
351 //
352 // The last three blocks are never deallocated or touched.
353
354 // Add MemoryRangeHeader to the end of the memory region, indicating that
355 // the space after the block of memory is allocated. This is block #3.
356 MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
357 Mem3->ThisAllocated = 1;
358 Mem3->PrevAllocated = 0;
359 Mem3->BlockSize = 0;
360
361 /// Add a tiny free region so that the free list always has one entry.
362 FreeRangeHeader *Mem2 =
363 (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
364 Mem2->ThisAllocated = 0;
365 Mem2->PrevAllocated = 1;
366 Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
367 Mem2->SetEndOfBlockSizeMarker();
368 Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
369 Mem2->Next = Mem2;
370
371 /// Add a tiny allocated region so that Mem2 is never coalesced away.
372 MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
373 Mem1->ThisAllocated = 1;
374 Mem1->PrevAllocated = 0;
375 Mem1->BlockSize = (char*)Mem2 - (char*)Mem1;
376
377 // Add a FreeRangeHeader to the start of the function body region, indicating
378 // that the space is free. Mark the previous block allocated so we never look
379 // at it.
380 FreeRangeHeader *Mem0 = (FreeRangeHeader*)CurStubPtr;
381 Mem0->ThisAllocated = 0;
382 Mem0->PrevAllocated = 1;
383 Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
384 Mem0->SetEndOfBlockSizeMarker();
385 Mem0->AddToFreeList(Mem2);
386
387 // Start out with the freelist pointing to Mem0.
388 FreeMemoryList = Mem0;
389
390 // Allocate the GOT.
391 GOTBase = NULL;
392 if (useGOT) GOTBase = new unsigned char[sizeof(void*) * 8192];
393 }
394
395 JITMemoryManager::~JITMemoryManager() {
396 for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
397 sys::Memory::ReleaseRWX(Blocks[i]);
398
399 delete[] GOTBase;
400 Blocks.clear();
401 }
402
403 unsigned char *JITMemoryManager::allocateStub(unsigned StubSize,
404 unsigned Alignment) {
405 CurStubPtr -= StubSize;
406 CurStubPtr = (unsigned char*)(((intptr_t)CurStubPtr) &
407 ~(intptr_t)(Alignment-1));
408 if (CurStubPtr < StubBase) {
409 // FIXME: allocate a new block
410 cerr << "JIT ran out of memory for function stubs!\n";
411 abort();
412 }
413 return CurStubPtr;
414 }
415
416 sys::MemoryBlock JITMemoryManager::getNewMemoryBlock(unsigned size) {
417 // Allocate a new block close to the last one.
418 const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.front();
419 std::string ErrMsg;
420 sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg);
421 if (B.base() == 0) {
422 cerr << "Allocation failed when allocating new memory in the JIT\n";
423 cerr << ErrMsg << "\n";
424 abort();
425 }
426 Blocks.push_back(B);
427 return B;
428 }
42939
43040 //===----------------------------------------------------------------------===//
43141 // JIT lazy compilation code.
503113 }
504114
505115 /// getGOTIndexForAddress - Return a new or existing index in the GOT for
506 /// and address. This function only manages slots, it does not manage the
116 /// an address. This function only manages slots, it does not manage the
507117 /// contents of the slots or the memory associated with the GOT.
508 unsigned getGOTIndexForAddr(void* addr);
118 unsigned getGOTIndexForAddr(void *addr);
509119
510120 /// JITCompilerFn - This function is called to resolve a stub to a compiled
511121 /// address. If the LLVM Function corresponding to the stub has not yet
596206 revGOTMap[addr] = idx;
597207 DOUT << "Adding GOT entry " << idx
598208 << " for addr " << addr << "\n";
599 // ((void**)MemMgr.getGOTBase())[idx] = addr;
600209 }
601210 return idx;
602211 }
668277 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is
669278 /// used to output functions to memory for execution.
670279 class JITEmitter : public MachineCodeEmitter {
671 JITMemoryManager MemMgr;
280 JITMemoryManager *MemMgr;
672281
673282 // When outputting a function stub in the context of some other function, we
674283 // save BufferBegin/BufferEnd/CurBufferPtr here.
702311 /// Resolver - This contains info about the currently resolved functions.
703312 JITResolver Resolver;
704313 public:
705 JITEmitter(JIT &jit)
706 : MemMgr(jit.getJITInfo().needsGOT()), Resolver(jit) {
707 if (MemMgr.isManagingGOT()) DOUT << "JIT is managing a GOT\n";
314 JITEmitter(JIT &jit) : Resolver(jit) {
315 MemMgr = JITMemoryManager::CreateDefaultMemManager();
316 if (jit.getJITInfo().needsGOT()) {
317 MemMgr->AllocateGOT();
318 DOUT << "JIT is managing a GOT\n";
319 }
320 }
321 ~JITEmitter() {
322 delete MemMgr;
708323 }
709324
710325 JITResolver &getJITResolver() { return Resolver; }
741356 /// deallocateMemForFunction - Deallocate all memory for the specified
742357 /// function body.
743358 void deallocateMemForFunction(Function *F) {
744 MemMgr.deallocateMemForFunction(F);
359 MemMgr->deallocateMemForFunction(F);
745360 }
746361 private:
747362 void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub);
782397
783398 void JITEmitter::startFunction(MachineFunction &F) {
784399 uintptr_t ActualSize;
785 BufferBegin = CurBufferPtr = MemMgr.startFunctionBody(ActualSize);
400 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
401 ActualSize);
786402 BufferEnd = BufferBegin+ActualSize;
787403
788404 // Ensure the constant pool/jump table info is at least 4-byte aligned.
813429 (unsigned char *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction());
814430 unsigned char *FnEnd = CurBufferPtr;
815431
816 MemMgr.endFunctionBody(F.getFunction(), BufferBegin, FnEnd);
432 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, FnEnd);
817433 NumBytes += FnEnd-FnStart;
818434
819435 if (!Relocations.empty()) {
846462
847463 // if we are managing the GOT and the relocation wants an index,
848464 // give it one
849 if (MemMgr.isManagingGOT() && MR.isGOTRelative()) {
465 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) {
850466 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr);
851467 MR.setGOTIndex(idx);
852 if (((void**)MemMgr.getGOTBase())[idx] != ResultPtr) {
468 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) {
853469 DOUT << "GOT was out of date for " << ResultPtr
854 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx]
470 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx]
855471 << "\n";
856 ((void**)MemMgr.getGOTBase())[idx] = ResultPtr;
472 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr;
857473 }
858474 }
859475 }
860476
861477 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0],
862 Relocations.size(), MemMgr.getGOTBase());
478 Relocations.size(), MemMgr->getGOTBase());
863479 }
864480
865481 // Update the GOT entry for F to point to the new code.
866 if (MemMgr.isManagingGOT()) {
482 if (MemMgr->isManagingGOT()) {
867483 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin);
868 if (((void**)MemMgr.getGOTBase())[idx] != (void*)BufferBegin) {
484 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) {
869485 DOUT << "GOT was out of date for " << (void*)BufferBegin
870 << " pointing at " << ((void**)MemMgr.getGOTBase())[idx] << "\n";
871 ((void**)MemMgr.getGOTBase())[idx] = (void*)BufferBegin;
486 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] << "\n";
487 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin;
872488 }
873489 }
874490
975591 SavedBufferEnd = BufferEnd;
976592 SavedCurBufferPtr = CurBufferPtr;
977593
978 BufferBegin = CurBufferPtr = MemMgr.allocateStub(StubSize, Alignment);
594 BufferBegin = CurBufferPtr = MemMgr->allocateStub(StubSize, Alignment);
979595 BufferEnd = BufferBegin+StubSize+1;
980596 }
981597
0 //===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file was developed by Chris Lattner and is distributed under
5 // the University of Illinois Open Source License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the DefaultJITMemoryManager class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ExecutionEngine/JITMemoryManager.h"
14 #include "llvm/Support/Compiler.h"
15 #include "llvm/System/Memory.h"
16 #include
17 #include
18 using namespace llvm;
19
20
21 JITMemoryManager::~JITMemoryManager() {}
22
23 //===----------------------------------------------------------------------===//
24 // Memory Block Implementation.
25 //===----------------------------------------------------------------------===//
26
27 namespace {
28 /// MemoryRangeHeader - For a range of memory, this is the header that we put
29 /// on the block of memory. It is carefully crafted to be one word of memory.
30 /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader
31 /// which starts with this.
32 struct FreeRangeHeader;
33 struct MemoryRangeHeader {
34 /// ThisAllocated - This is true if this block is currently allocated. If
35 /// not, this can be converted to a FreeRangeHeader.
36 unsigned ThisAllocated : 1;
37
38 /// PrevAllocated - Keep track of whether the block immediately before us is
39 /// allocated. If not, the word immediately before this header is the size
40 /// of the previous block.
41 unsigned PrevAllocated : 1;
42
43 /// BlockSize - This is the size in bytes of this memory block,
44 /// including this header.
45 uintptr_t BlockSize : (sizeof(intptr_t)*8 - 2);
46
47
48 /// getBlockAfter - Return the memory block immediately after this one.
49 ///
50 MemoryRangeHeader &getBlockAfter() const {
51 return *(MemoryRangeHeader*)((char*)this+BlockSize);
52 }
53
54 /// getFreeBlockBefore - If the block before this one is free, return it,
55 /// otherwise return null.
56 FreeRangeHeader *getFreeBlockBefore() const {
57 if (PrevAllocated) return 0;
58 intptr_t PrevSize = ((intptr_t *)this)[-1];
59 return (FreeRangeHeader*)((char*)this-PrevSize);
60 }
61
62 /// FreeBlock - Turn an allocated block into a free block, adjusting
63 /// bits in the object headers, and adding an end of region memory block.
64 FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList);
65
66 /// TrimAllocationToSize - If this allocated block is significantly larger
67 /// than NewSize, split it into two pieces (where the former is NewSize
68 /// bytes, including the header), and add the new block to the free list.
69 FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList,
70 uint64_t NewSize);
71 };
72
73 /// FreeRangeHeader - For a memory block that isn't already allocated, this
74 /// keeps track of the current block and has a pointer to the next free block.
75 /// Free blocks are kept on a circularly linked list.
76 struct FreeRangeHeader : public MemoryRangeHeader {
77 FreeRangeHeader *Prev;
78 FreeRangeHeader *Next;
79
80 /// getMinBlockSize - Get the minimum size for a memory block. Blocks
81 /// smaller than this size cannot be created.
82 static unsigned getMinBlockSize() {
83 return sizeof(FreeRangeHeader)+sizeof(intptr_t);
84 }
85
86 /// SetEndOfBlockSizeMarker - The word at the end of every free block is
87 /// known to be the size of the free block. Set it for this block.
88 void SetEndOfBlockSizeMarker() {
89 void *EndOfBlock = (char*)this + BlockSize;
90 ((intptr_t *)EndOfBlock)[-1] = BlockSize;
91 }
92
93 FreeRangeHeader *RemoveFromFreeList() {
94 assert(Next->Prev == this && Prev->Next == this && "Freelist broken!");
95 Next->Prev = Prev;
96 return Prev->Next = Next;
97 }
98
99 void AddToFreeList(FreeRangeHeader *FreeList) {
100 Next = FreeList;
101 Prev = FreeList->Prev;
102 Prev->Next = this;
103 Next->Prev = this;
104 }
105
106 /// GrowBlock - The block after this block just got deallocated. Merge it
107 /// into the current block.
108 void GrowBlock(uintptr_t NewSize);
109
110 /// AllocateBlock - Mark this entire block allocated, updating freelists
111 /// etc. This returns a pointer to the circular free-list.
112 FreeRangeHeader *AllocateBlock();
113 };
114 }
115
116
117 /// AllocateBlock - Mark this entire block allocated, updating freelists
118 /// etc. This returns a pointer to the circular free-list.
119 FreeRangeHeader *FreeRangeHeader::AllocateBlock() {
120 assert(!ThisAllocated && !getBlockAfter().PrevAllocated &&
121 "Cannot allocate an allocated block!");
122 // Mark this block allocated.
123 ThisAllocated = 1;
124 getBlockAfter().PrevAllocated = 1;
125
126 // Remove it from the free list.
127 return RemoveFromFreeList();
128 }
129
130 /// FreeBlock - Turn an allocated block into a free block, adjusting
131 /// bits in the object headers, and adding an end of region memory block.
132 /// If possible, coalesce this block with neighboring blocks. Return the
133 /// FreeRangeHeader to allocate from.
134 FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) {
135 MemoryRangeHeader *FollowingBlock = &getBlockAfter();
136 assert(ThisAllocated && "This block is already allocated!");
137 assert(FollowingBlock->PrevAllocated && "Flags out of sync!");
138
139 FreeRangeHeader *FreeListToReturn = FreeList;
140
141 // If the block after this one is free, merge it into this block.
142 if (!FollowingBlock->ThisAllocated) {
143 FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock;
144 // "FreeList" always needs to be a valid free block. If we're about to
145 // coalesce with it, update our notion of what the free list is.
146 if (&FollowingFreeBlock == FreeList) {
147 FreeList = FollowingFreeBlock.Next;
148 FreeListToReturn = 0;
149 assert(&FollowingFreeBlock != FreeList && "No tombstone block?");
150 }
151 FollowingFreeBlock.RemoveFromFreeList();
152
153 // Include the following block into this one.
154 BlockSize += FollowingFreeBlock.BlockSize;
155 FollowingBlock = &FollowingFreeBlock.getBlockAfter();
156
157 // Tell the block after the block we are coalescing that this block is
158 // allocated.
159 FollowingBlock->PrevAllocated = 1;
160 }
161
162 assert(FollowingBlock->ThisAllocated && "Missed coalescing?");
163
164 if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) {
165 PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize);
166 return FreeListToReturn ? FreeListToReturn : PrevFreeBlock;
167 }
168
169 // Otherwise, mark this block free.
170 FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this;
171 FollowingBlock->PrevAllocated = 0;
172 FreeBlock.ThisAllocated = 0;
173
174 // Link this into the linked list of free blocks.
175 FreeBlock.AddToFreeList(FreeList);
176
177 // Add a marker at the end of the block, indicating the size of this free
178 // block.
179 FreeBlock.SetEndOfBlockSizeMarker();
180 return FreeListToReturn ? FreeListToReturn : &FreeBlock;
181 }
182
183 /// GrowBlock - The block after this block just got deallocated. Merge it
184 /// into the current block.
185 void FreeRangeHeader::GrowBlock(uintptr_t NewSize) {
186 assert(NewSize > BlockSize && "Not growing block?");
187 BlockSize = NewSize;
188 SetEndOfBlockSizeMarker();
189 getBlockAfter().PrevAllocated = 0;
190 }
191
192 /// TrimAllocationToSize - If this allocated block is significantly larger
193 /// than NewSize, split it into two pieces (where the former is NewSize
194 /// bytes, including the header), and add the new block to the free list.
195 FreeRangeHeader *MemoryRangeHeader::
196 TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) {
197 assert(ThisAllocated && getBlockAfter().PrevAllocated &&
198 "Cannot deallocate part of an allocated block!");
199
200 // Round up size for alignment of header.
201 unsigned HeaderAlign = __alignof(FreeRangeHeader);
202 NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1);
203
204 // Size is now the size of the block we will remove from the start of the
205 // current block.
206 assert(NewSize <= BlockSize &&
207 "Allocating more space from this block than exists!");
208
209 // If splitting this block will cause the remainder to be too small, do not
210 // split the block.
211 if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize())
212 return FreeList;
213
214 // Otherwise, we splice the required number of bytes out of this block, form
215 // a new block immediately after it, then mark this block allocated.
216 MemoryRangeHeader &FormerNextBlock = getBlockAfter();
217
218 // Change the size of this block.
219 BlockSize = NewSize;
220
221 // Get the new block we just sliced out and turn it into a free block.
222 FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter();
223 NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock;
224 NewNextBlock.ThisAllocated = 0;
225 NewNextBlock.PrevAllocated = 1;
226 NewNextBlock.SetEndOfBlockSizeMarker();
227 FormerNextBlock.PrevAllocated = 0;
228 NewNextBlock.AddToFreeList(FreeList);
229 return &NewNextBlock;
230 }
231
232 //===----------------------------------------------------------------------===//
233 // Memory Block Implementation.
234 //===----------------------------------------------------------------------===//
235
236 namespace {
237 /// DefaultJITMemoryManager - Manage memory for the JIT code generation.
238 /// This splits a large block of MAP_NORESERVE'd memory into two
239 /// sections, one for function stubs, one for the functions themselves. We
240 /// have to do this because we may need to emit a function stub while in the
241 /// middle of emitting a function, and we don't know how large the function we
242 /// are emitting is.
243 class VISIBILITY_HIDDEN DefaultJITMemoryManager : public JITMemoryManager {
244 std::vector Blocks; // Memory blocks allocated by the JIT
245 FreeRangeHeader *FreeMemoryList; // Circular list of free blocks.
246
247 // When emitting code into a memory block, this is the block.
248 MemoryRangeHeader *CurBlock;
249
250 unsigned char *CurStubPtr, *StubBase;
251 unsigned char *GOTBase; // Target Specific reserved memory
252
253 // Centralize memory block allocation.
254 sys::MemoryBlock getNewMemoryBlock(unsigned size);
255
256 std::map FunctionBlocks;
257 public:
258 DefaultJITMemoryManager();
259 ~DefaultJITMemoryManager();
260
261 void AllocateGOT();
262
263 unsigned char *allocateStub(unsigned StubSize, unsigned Alignment);
264
265 /// startFunctionBody - When a function starts, allocate a block of free
266 /// executable memory, returning a pointer to it and its actual size.
267 unsigned char *startFunctionBody(const Function *F, uintptr_t &ActualSize) {
268 CurBlock = FreeMemoryList;
269
270 // Allocate the entire memory block.
271 FreeMemoryList = FreeMemoryList->AllocateBlock();
272 ActualSize = CurBlock->BlockSize-sizeof(MemoryRangeHeader);
273 return (unsigned char *)(CurBlock+1);
274 }
275
276 /// endFunctionBody - The function F is now allocated, and takes the memory
277 /// in the range [FunctionStart,FunctionEnd).
278 void endFunctionBody(const Function *F, unsigned char *FunctionStart,
279 unsigned char *FunctionEnd) {
280 assert(FunctionEnd > FunctionStart);
281 assert(FunctionStart == (unsigned char *)(CurBlock+1) &&
282 "Mismatched function start/end!");
283
284 uintptr_t BlockSize = FunctionEnd - (unsigned char *)CurBlock;
285 FunctionBlocks[F] = CurBlock;
286
287 // Release the memory at the end of this block that isn't needed.
288 FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize);
289 }
290
291 unsigned char *getGOTBase() const {
292 return GOTBase;
293 }
294
295 /// deallocateMemForFunction - Deallocate all memory for the specified
296 /// function body.
297 void deallocateMemForFunction(const Function *F) {
298 std::map::iterator
299 I = FunctionBlocks.find(F);
300 if (I == FunctionBlocks.end()) return;
301
302 // Find the block that is allocated for this function.
303 MemoryRangeHeader *MemRange = I->second;
304 assert(MemRange->ThisAllocated && "Block isn't allocated!");
305
306 // Fill the buffer with garbage!
307 #ifndef NDEBUG
308 memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
309 #endif
310
311 // Free the memory.
312 FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
313
314 // Finally, remove this entry from FunctionBlocks.
315 FunctionBlocks.erase(I);
316 }
317 };
318 }
319
320 DefaultJITMemoryManager::DefaultJITMemoryManager() {
321 // Allocate a 16M block of memory for functions.
322 sys::MemoryBlock MemBlock = getNewMemoryBlock(16 << 20);
323
324 unsigned char *MemBase = reinterpret_cast(MemBlock.base());
325
326 // Allocate stubs backwards from the base, allocate functions forward
327 // from the base.
328 StubBase = MemBase;
329 CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards.
330
331 // We set up the memory chunk with 4 mem regions, like this:
332 // [ START
333 // [ Free #0 ] -> Large space to allocate functions from.
334 // [ Allocated #1 ] -> Tiny space to separate regions.
335 // [ Free #2 ] -> Tiny space so there is always at least 1 free block.
336 // [ Allocated #3 ] -> Tiny space to prevent looking past end of block.
337 // END ]
338 //
339 // The last three blocks are never deallocated or touched.
340
341 // Add MemoryRangeHeader to the end of the memory region, indicating that
342 // the space after the block of memory is allocated. This is block #3.
343 MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1;
344 Mem3->ThisAllocated = 1;
345 Mem3->PrevAllocated = 0;
346 Mem3->BlockSize = 0;
347
348 /// Add a tiny free region so that the free list always has one entry.
349 FreeRangeHeader *Mem2 =
350 (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize());
351 Mem2->ThisAllocated = 0;
352 Mem2->PrevAllocated = 1;
353 Mem2->BlockSize = FreeRangeHeader::getMinBlockSize();
354 Mem2->SetEndOfBlockSizeMarker();
355 Mem2->Prev = Mem2; // Mem2 *is* the free list for now.
356 Mem2->Next = Mem2;
357
358 /// Add a tiny allocated region so that Mem2 is never coalesced away.
359 MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1;
360 Mem1->ThisAllocated = 1;
361 Mem1->PrevAllocated = 0;
362 Mem1->BlockSize = (char*)Mem2 - (char*)Mem1;
363
364 // Add a FreeRangeHeader to the start of the function body region, indicating
365 // that the space is free. Mark the previous block allocated so we never look
366 // at it.
367 FreeRangeHeader *Mem0 = (FreeRangeHeader*)CurStubPtr;
368 Mem0->ThisAllocated = 0;
369 Mem0->PrevAllocated = 1;
370 Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
371 Mem0->SetEndOfBlockSizeMarker();
372 Mem0->AddToFreeList(Mem2);
373
374 // Start out with the freelist pointing to Mem0.
375 FreeMemoryList = Mem0;
376
377 GOTBase = NULL;
378 }
379
380 void DefaultJITMemoryManager::AllocateGOT() {
381 assert(GOTBase == 0 && "Cannot allocate the got multiple times");
382 GOTBase = new unsigned char[sizeof(void*) * 8192];
383 HasGOT = true;
384 }
385
386
387 DefaultJITMemoryManager::~DefaultJITMemoryManager() {
388 for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
389 sys::Memory::ReleaseRWX(Blocks[i]);
390
391 delete[] GOTBase;
392 Blocks.clear();
393 }
394
395 unsigned char *DefaultJITMemoryManager::allocateStub(unsigned StubSize,
396 unsigned Alignment) {
397 CurStubPtr -= StubSize;
398 CurStubPtr = (unsigned char*)(((intptr_t)CurStubPtr) &
399 ~(intptr_t)(Alignment-1));
400 if (CurStubPtr < StubBase) {
401 // FIXME: allocate a new block
402 fprintf(stderr, "JIT ran out of memory for function stubs!\n");
403 abort();
404 }
405 return CurStubPtr;
406 }
407
408 sys::MemoryBlock DefaultJITMemoryManager::getNewMemoryBlock(unsigned size) {
409 // Allocate a new block close to the last one.
410 const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.front();
411 std::string ErrMsg;
412 sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg);
413 if (B.base() == 0) {
414 fprintf(stderr,
415 "Allocation failed when allocating new memory in the JIT\n%s\n",
416 ErrMsg.c_str());
417 abort();
418 }
419 Blocks.push_back(B);
420 return B;
421 }
422
423
424 JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() {
425 return new DefaultJITMemoryManager();
426 }