llvm.org GIT mirror llvm / d159fdd
CachePruning: Allow limiting the number of files in the cache directory. The default limit is 1000000 but it can be configured with a cache policy. The motivation is that some filesystems (notably ext4) have a limit on the number of files that can be contained in a directory (separate from the inode limit). Differential Revision: https://reviews.llvm.org/D40327 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@318857 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 2 years ago
2 changed file(s) with 39 addition(s) and 20 deletion(s). Raw diff Collapse all Expand all
4545 /// of available space on the disk will be reduced to the amount of available
4646 /// space. A value of 0 disables the absolute size-based pruning.
4747 uint64_t MaxSizeBytes = 0;
48
49 /// The maximum number of files in the cache directory. A value of 0 disables
50 /// the number of files based pruning.
51 ///
52 /// This defaults to 1000000 because with that many files there are
53 /// diminishing returns on the effectiveness of the cache, and some file
54 /// systems have a limit on how many files can be contained in a directory
55 /// (notably ext4, which is limited to around 6000000 files).
56 uint64_t MaxSizeFiles = 1000000;
4857 };
4958
5059 /// Parse the given string as a cache pruning policy. Defaults are taken from a
112112 return make_error("'" + Value + "' not an integer",
113113 inconvertibleErrorCode());
114114 Policy.MaxSizeBytes = Size * Mult;
115 } else if (Key == "cache_size_files") {
116 if (Value.getAsInteger(0, Policy.MaxSizeFiles))
117 return make_error("'" + Value + "' not an integer",
118 inconvertibleErrorCode());
115119 } else {
116120 return make_error("Unknown key: '" + Key + "'",
117121 inconvertibleErrorCode());
140144
141145 if (Policy.Expiration == seconds(0) &&
142146 Policy.MaxSizePercentageOfAvailableSpace == 0 &&
143 Policy.MaxSizeBytes == 0) {
147 Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
144148 DEBUG(dbgs() << "No pruning settings set, exit early\n");
145149 // Nothing will be pruned, early exit
146150 return false;
177181 // notice at the same time that the timestamp is out-of-date.
178182 writeTimestampFile(TimestampFile);
179183 }
180
181 bool ShouldComputeSize =
182 (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
183184
184185 // Keep track of space. Needs to be kept ordered by size for determinism.
185186 std::set> FileSizes;
210211 // If the file hasn't been used recently enough, delete it
211212 const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
212213 auto FileAge = CurrentTime - FileAccessTime;
213 if (FileAge > Policy.Expiration) {
214 if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
214215 DEBUG(dbgs() << "Remove " << File->path() << " ("
215216 << duration_cast(FileAge).count() << "s old)\n");
216217 sys::fs::remove(File->path());
218219 }
219220
220221 // Leave it here for now, but add it to the list of size-based pruning.
221 if (!ShouldComputeSize)
222 continue;
223222 TotalSize += StatusOrErr->getSize();
224223 FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
225224 }
226225
226 auto FileAndSize = FileSizes.rbegin();
227 size_t NumFiles = FileSizes.size();
228
229 auto RemoveCacheFile = [&]() {
230 // Remove the file.
231 sys::fs::remove(FileAndSize->second);
232 // Update size
233 TotalSize -= FileAndSize->first;
234 NumFiles--;
235 DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
236 << FileAndSize->first << "), new occupancy is " << TotalSize
237 << "%\n");
238 ++FileAndSize;
239 };
240
241 // Prune for number of files.
242 if (Policy.MaxSizeFiles)
243 while (NumFiles > Policy.MaxSizeFiles)
244 RemoveCacheFile();
245
227246 // Prune for size now if needed
228 if (ShouldComputeSize) {
247 if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
229248 auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
230249 if (!ErrOrSpaceInfo) {
231250 report_fatal_error("Can't get available size");
245264 << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
246265 << "%, " << Policy.MaxSizeBytes << " bytes\n");
247266
248 auto FileAndSize = FileSizes.rbegin();
249 // Remove the oldest accessed files first, till we get below the threshold
250 while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
251 // Remove the file.
252 sys::fs::remove(FileAndSize->second);
253 // Update size
254 TotalSize -= FileAndSize->first;
255 DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
256 << FileAndSize->first << "), new occupancy is " << TotalSize
257 << "%\n");
258 ++FileAndSize;
259 }
267 // Remove the oldest accessed files first, till we get below the threshold.
268 while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
269 RemoveCacheFile();
260270 }
261271 return true;
262272 }