llvm.org GIT mirror llvm / 22706dc
Add Cache Pruning support Incremental LTO will usea cache to store object files. This patch handles the pruning part of the cache, exposing a few knobs: - Pruning interval: the implementation keeps a "timestamp" file in the directory and will scan it only after a given interval since the last modification of the timestamp file. This is for performance purpose, we don't want to scan continuously the folder. - Entry expiration: this is the time after which a file that hasn't been used is remove from the cache. - Maximum size: expressed in percentage of the available disk space, it helps to avoid that we blow up the disk space. http://reviews.llvm.org/D18422 From: Mehdi Amini <mehdi.amini@apple.com> git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@265209 91177308-0d34-0410-b5e6-96231b3b80d8 Mehdi Amini 4 years ago
3 changed file(s) with 200 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
0 //=- CachePruning.h - Helper to manage the pruning of a cache dir -*- C++ -*-=//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements pruning of a directory intended for cache storage, using
10 // various policies.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_SUPPORT_CACHE_PRUNING_H
15 #define LLVM_SUPPORT_CACHE_PRUNING_H
16
17 #include "llvm/ADT/StringRef.h"
18
19 namespace llvm {
20
21 /// Handle pruning a directory provided a path and some options to control what
22 /// to prune.
23 class CachePruning {
24 public:
25 /// Prepare to prune \p Path.
26 CachePruning(StringRef Path) : Path(Path) {}
27
28 /// Define the pruning interval. This is intended to be used to avoid scanning
29 /// the directory too often. It does not impact the decision of which file to
30 /// prune. A value of 0 forces the scan to occurs.
31 CachePruning &setPruningInterval(int PruningInterval) {
32 Interval = PruningInterval;
33 return *this;
34 }
35
36 /// Define the expiration for a file. When a file hasn't been accessed for
37 /// \p ExpireAfter seconds, it is removed from the cache. A value of 0 disable
38 /// the expiration-based pruning.
39 CachePruning &setEntryExpiration(unsigned ExpireAfter) {
40 Expiration = ExpireAfter;
41 return *this;
42 }
43
44 /// Define the maximum size for the cache directory, in terms of percentage of
45 /// the available space on the the disk. Set to 100 to indicate no limit, 50
46 /// to indicate that the cache size will not be left over half the
47 /// available disk space. A value over 100 will be reduced to 100. A value of
48 /// 0 disable the size-based pruning.
49 CachePruning &setMaxSize(unsigned Percentage) {
50 PercentageOfAvailableSpace = std::min(100u, Percentage);
51 return *this;
52 }
53
54 /// Peform pruning using the supplied options, returns true if pruning
55 /// occured, i.e. if PruningInterval was expired.
56 bool prune();
57
58 private:
59 // Options that matches the setters above.
60 std::string Path;
61 unsigned Expiration = 0;
62 unsigned Interval = 0;
63 unsigned PercentageOfAvailableSpace = 0;
64 };
65
66 } // namespace llvm
67
68 #endif
3434 Allocator.cpp
3535 BlockFrequency.cpp
3636 BranchProbability.cpp
37 CachePruning.cpp
3738 circular_raw_ostream.cpp
3839 COM.cpp
3940 CommandLine.cpp
0 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
1 //
2 // The LLVM Compiler Infrastructure
3 //
4 // This file is distributed under the University of Illinois Open Source
5 // License. See LICENSE.TXT for details.
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the pruning of a directory based on least recently used.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/Support/CachePruning.h"
14
15 #include "llvm/Support/FileSystem.h"
16 #include "llvm/Support/Path.h"
17 #include "llvm/Support/raw_ostream.h"
18
19 #include
20
21 using namespace llvm;
22
23 /// Write a new timestamp file with the given path. This is used for the pruning
24 /// interval option.
25 static void writeTimestampFile(StringRef TimestampFile) {
26 std::error_code EC;
27 raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
28 }
29
30 /// Prune the cache of files that haven't been accessed in a long time.
31 bool CachePruning::prune() {
32 SmallString<128> TimestampFile(Path);
33 sys::path::append(TimestampFile, "llvmcache.timestamp");
34
35 if (Expiration == 0 && PercentageOfAvailableSpace == 0)
36 // Nothing will be pruned, early exit
37 return false;
38
39 // Try to stat() the timestamp file.
40 sys::fs::file_status FileStatus;
41 sys::TimeValue CurrentTime = sys::TimeValue::now();
42 if (sys::fs::status(TimestampFile, FileStatus)) {
43 if (errno == ENOENT) {
44 // If the timestamp file wasn't there, create one now.
45 writeTimestampFile(TimestampFile);
46 } else {
47 // Unknown error?
48 return false;
49 }
50 } else {
51 if (Interval) {
52 // Check whether the time stamp is older than our pruning interval.
53 // If not, do nothing.
54 sys::TimeValue TimeStampModTime = FileStatus.getLastModificationTime();
55 auto TimeInterval = sys::TimeValue(sys::TimeValue::SecondsType(Interval));
56 if (CurrentTime - TimeStampModTime <= TimeInterval)
57 return false;
58 }
59 // Write a new timestamp file so that nobody else attempts to prune.
60 // There is a benign race condition here, if two processes happen to
61 // notice at the same time that the timestamp is out-of-date.
62 writeTimestampFile(TimestampFile);
63 }
64
65 bool ShouldComputeSize = (PercentageOfAvailableSpace > 0);
66
67 // Keep track of space
68 std::set> FileSizes;
69 uint64_t TotalSize = 0;
70 // Helper to add a path to the set of files to consider for size-based
71 // pruning, sorted by last accessed time.
72 auto AddToFileListForSizePruning =
73 [&](StringRef Path, sys::TimeValue FileAccessTime) {
74 if (!ShouldComputeSize)
75 return;
76 TotalSize += FileStatus.getSize();
77 FileSizes.insert(
78 std::make_pair(FileAccessTime.seconds(), std::string(Path)));
79 };
80
81 // Walk the entire directory cache, looking for unused files.
82 std::error_code EC;
83 SmallString<128> CachePathNative;
84 sys::path::native(Path, CachePathNative);
85 auto TimeExpiration = sys::TimeValue(sys::TimeValue::SecondsType(Expiration));
86 // Walk all of the files within this directory.
87 for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
88 File != FileEnd && !EC; File.increment(EC)) {
89 // Do not touch the timestamp.
90 if (File->path() == TimestampFile)
91 continue;
92
93 // Look at this file. If we can't stat it, there's nothing interesting
94 // there.
95 if (sys::fs::status(File->path(), FileStatus))
96 continue;
97
98 // If the file hasn't been used recently enough, delete it
99 sys::TimeValue FileAccessTime = FileStatus.getLastAccessedTime();
100 if (CurrentTime - FileAccessTime > TimeExpiration) {
101 sys::fs::remove(File->path());
102 continue;
103 }
104
105 // Leave it here for now, but add it to the list of size-based pruning.
106 AddToFileListForSizePruning(File->path(), FileAccessTime);
107 }
108
109 // Prune for size now if needed
110 if (ShouldComputeSize) {
111 auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
112 if (!ErrOrSpaceInfo) {
113 report_fatal_error("Can't get available size");
114 }
115 sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
116 auto AvailableSpace = TotalSize + SpaceInfo.free;
117 auto FileAndSize = FileSizes.rbegin();
118 // Remove the oldest accessed files first, till we get below the threshold
119 while (((100 * TotalSize) / AvailableSpace) > PercentageOfAvailableSpace &&
120 FileAndSize != FileSizes.rend()) {
121 // Remove the file.
122 sys::fs::remove(FileAndSize->second);
123 // Update size
124 TotalSize -= FileAndSize->first;
125 ++FileAndSize;
126 }
127 }
128 return true;
129 }