llvm.org GIT mirror llvm / f04cdf9
[SectionMemoryManager] Make better use of virtual memory Summary: On Windows, the allocation granularity can be significantly larger than a page (64K), so with many small objects, just clearing the FreeMem list rapidly leaks quite a bit of virtual memory space (if not rss). Fix that by only removing those parts of the FreeMem blocks that overlap pages for which we are applying memory permissions, rather than dropping the FreeMem blocks entirely. Reviewers: lhames Subscribers: llvm-commits Differential Revision: http://reviews.llvm.org/D15202 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@255760 91177308-0d34-0410-b5e6-96231b3b80d8 Keno Fischer 3 years ago
3 changed file(s) with 89 addition(s) and 43 deletion(s). Raw diff Collapse all Expand all
8282 virtual void invalidateInstructionCache();
8383
8484 private:
85 struct FreeMemBlock {
86 // The actual block of free memory
87 sys::MemoryBlock Free;
88 // If there is a pending allocation from the same reservation right before
89 // this block, store it's index in PendingMem, to be able to update the
90 // pending region if part of this block is allocated, rather than having to
91 // create a new one
92 unsigned PendingPrefixIndex;
93 };
94
8595 struct MemoryGroup {
86 // PendingMem contains all allocated memory blocks
87 // which have not yet had their permissions set. Note
88 // that this tracks memory blocks that have been given to
89 // this memory manager by the system, not those
90 // given out to the user. In particular, the memory manager
91 // will give out subblocks of these MemoryBlocks in response
92 // to user requests. We track which subblocks have not beeen
93 // given out yet in `FreeMem`.
94 SmallVector PendingMem;
95 SmallVector FreeMem;
96 // PendingMem contains all blocks of memory (subblocks of AllocatedMem)
97 // which have not yet had their permissions applied, but have been given
98 // out to the user. FreeMem contains all block of memory, which have
99 // neither had their permissions applied, nor been given out to the user.
100 SmallVector PendingMem;
101 SmallVector FreeMem;
96102
97 // All allocated memory blocks that have had their permissions
98 // set (i.e. that have been finalized). Because of this, we may
99 // not give out subblocks of this memory to the user anymore,
100 // even if those subblocks have not been previously given out.
101 SmallVector AllocatedMem;
103 // All memory blocks that have been requested from the system
104 SmallVector AllocatedMem;
102105
103 sys::MemoryBlock Near;
106 sys::MemoryBlock Near;
104107 };
105108
106109 uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size,
117120 }
118121
119122 #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H
120
1414 #include "llvm/Config/config.h"
1515 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
1616 #include "llvm/Support/MathExtras.h"
17 #include "llvm/Support/Process.h"
1718
1819 namespace llvm {
1920
4748
4849 // Look in the list of free memory regions and use a block there if one
4950 // is available.
50 for (sys::MemoryBlock &MB : MemGroup.FreeMem) {
51 if (MB.size() >= RequiredSize) {
52 Addr = (uintptr_t)MB.base();
53 uintptr_t EndOfBlock = Addr + MB.size();
51 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
52 if (FreeMB.Free.size() >= RequiredSize) {
53 Addr = (uintptr_t)FreeMB.Free.base();
54 uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
5455 // Align the address.
5556 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
56 // Store cutted free memory block.
57 MB = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
57
58 if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
59 // The part of the block we're giving out to the user is now pending
60 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
61
62 // Remember this pending block, such that future allocations can just
63 // modify it rather than creating a new one
64 FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
65 } else {
66 sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
67 PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
68 }
69
70 // Remember how much free space is now left in this block
71 FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
5872 return (uint8_t*)Addr;
5973 }
6074 }
8296 // Save this address as the basis for our next request
8397 MemGroup.Near = MB;
8498
85 MemGroup.PendingMem.push_back(MB);
99 // Remember that we allocated this memory
100 MemGroup.AllocatedMem.push_back(MB);
86101 Addr = (uintptr_t)MB.base();
87102 uintptr_t EndOfBlock = Addr + MB.size();
88103
89104 // Align the address.
90105 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
106
107 // The part of the block we're giving out to the user is now pending
108 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
91109
92110 // The allocateMappedMemory may allocate much more memory than we need. In
93111 // this case, we store the unused memory as a free memory block.
94112 unsigned FreeSize = EndOfBlock-Addr-Size;
95 if (FreeSize > 16)
96 MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize));
113 if (FreeSize > 16) {
114 FreeMemBlock FreeMB;
115 FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize);
116 FreeMB.PendingPrefixIndex = (unsigned)-1;
117 MemGroup.FreeMem.push_back(FreeMB);
118 }
97119
98120 // Return aligned address
99121 return (uint8_t*)Addr;
103125 {
104126 // FIXME: Should in-progress permissions be reverted if an error occurs?
105127 std::error_code ec;
106
107 // Don't allow free memory blocks to be used after setting protection flags.
108 CodeMem.FreeMem.clear();
109128
110129 // Make code memory executable.
111130 ec = applyMemoryGroupPermissions(CodeMem,
137156 // relocations) will get to the data cache but not to the instruction cache.
138157 invalidateInstructionCache();
139158
140 // Now, remember that we have successfully applied the permissions to avoid
141 // having to apply them again.
142 CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end());
143 CodeMem.PendingMem.clear();
144
145 RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end());
146 RODataMem.PendingMem.clear();
147
148159 return false;
149160 }
161
162 static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
163 static const size_t PageSize = sys::Process::getPageSize();
164
165 size_t StartOverlap =
166 (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
167
168 size_t TrimmedSize = M.size();
169 TrimmedSize -= StartOverlap;
170 TrimmedSize -= TrimmedSize % PageSize;
171
172 sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize);
173
174 assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
175 assert((Trimmed.size() % PageSize) == 0);
176 assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size());
177
178 return Trimmed;
179 }
180
150181
151182 std::error_code
152183 SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
153184 unsigned Permissions) {
154
155185 for (sys::MemoryBlock &MB : MemGroup.PendingMem)
156186 if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions))
157187 return EC;
188
189 MemGroup.PendingMem.clear();
190
191 // Now go through free blocks and trim any of them that don't span the entire
192 // page because one of the pending blocks may have overlapped it.
193 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
194 FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
195 // We cleared the PendingMem list, so all these pointers are now invalid
196 FreeMB.PendingPrefixIndex = (unsigned)-1;
197 }
198
199 // Remove all blocks which are now empty
200 MemGroup.FreeMem.erase(
201 std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(),
202 [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }),
203 MemGroup.FreeMem.end());
158204
159205 return std::error_code();
160206 }
168214 for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
169215 for (sys::MemoryBlock &Block : Group->AllocatedMem)
170216 sys::Memory::releaseMappedMemory(Block);
171 for (sys::MemoryBlock &Block : Group->PendingMem)
172 sys::Memory::releaseMappedMemory(Block);
173217 }
174218 }
175219
176220 } // namespace llvm
177
151151
152152 std::error_code
153153 Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
154 static const size_t PageSize = Process::getPageSize();
154155 if (M.Address == nullptr || M.Size == 0)
155156 return std::error_code();
156157
159160
160161 int Protect = getPosixProtectionFlags(Flags);
161162
162 int Result = ::mprotect(M.Address, M.Size, Protect);
163 int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect);
163164 if (Result != 0)
164165 return std::error_code(errno, std::generic_category());
165166
179180 std::string *ErrMsg) {
180181 if (NumBytes == 0) return MemoryBlock();
181182
182 size_t PageSize = Process::getPageSize();
183 static const size_t PageSize = Process::getPageSize();
183184 size_t NumPages = (NumBytes+PageSize-1)/PageSize;
184185
185186 int fd = -1;