llvm.org GIT mirror llvm / 5cc49a2
Add a ThinLTO cache policy for controlling the maximum cache size in bytes. This is useful when an upper limit on the cache size needs to be controlled independently of the amount of the amount of free space. One use case is a machine with a large number of cache directories (e.g. a buildbot slave hosting a large number of independent build jobs). By imposing an upper size limit on each cache directory, users can more easily estimate the server's capacity. Differential Revision: https://reviews.llvm.org/D34547 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@306126 91177308-0d34-0410-b5e6-96231b3b80d8 Peter Collingbourne 3 years ago
4 changed file(s) with 84 addition(s) and 22 deletion(s). Raw diff Collapse all Expand all
176176 */
177177 void setMaxCacheSizeRelativeToAvailableSpace(unsigned Percentage) {
178178 if (Percentage)
179 CacheOptions.Policy.PercentageOfAvailableSpace = Percentage;
179 CacheOptions.Policy.MaxSizePercentageOfAvailableSpace = Percentage;
180180 }
181181
182182 /**@}*/
3838 /// available space on the the disk. Set to 100 to indicate no limit, 50 to
3939 /// indicate that the cache size will not be left over half the available disk
4040 /// space. A value over 100 will be reduced to 100. A value of 0 disables the
41 /// size-based pruning.
42 unsigned PercentageOfAvailableSpace = 75;
41 /// percentage size-based pruning.
42 unsigned MaxSizePercentageOfAvailableSpace = 75;
43
44 /// The maximum size for the cache directory in bytes. A value over the amount
45 /// of available space on the disk will be reduced to the amount of available
46 /// space. A value of 0 disables the absolute size-based pruning.
47 uint64_t MaxSizeBytes = 0;
4348 };
4449
4550 /// Parse the given string as a cache pruning policy. Defaults are taken from a
7878 return DurationOrErr.takeError();
7979 Policy.Expiration = *DurationOrErr;
8080 } else if (Key == "cache_size") {
81 if (Value.back() != '%')
82 return make_error("'" + Value + "' must be a percentage",
83 inconvertibleErrorCode());
84 StringRef SizeStr = Value.slice(0, Value.size() - 1);
81 if (Value.back() != '%')
82 return make_error("'" + Value + "' must be a percentage",
83 inconvertibleErrorCode());
84 StringRef SizeStr = Value.drop_back();
8585 uint64_t Size;
8686 if (SizeStr.getAsInteger(0, Size))
8787 return make_error("'" + SizeStr + "' not an integer",
9090 return make_error("'" + SizeStr +
9191 "' must be between 0 and 100",
9292 inconvertibleErrorCode());
93 Policy.PercentageOfAvailableSpace = Size;
93 Policy.MaxSizePercentageOfAvailableSpace = Size;
94 } else if (Key == "cache_size_bytes") {
95 uint64_t Mult = 1;
96 switch (Value.back()) {
97 case 'k':
98 Mult = 1024;
99 Value = Value.drop_back();
100 break;
101 case 'm':
102 Mult = 1024 * 1024;
103 Value = Value.drop_back();
104 break;
105 case 'g':
106 Mult = 1024 * 1024 * 1024;
107 Value = Value.drop_back();
108 break;
109 }
110 uint64_t Size;
111 if (Value.getAsInteger(0, Size))
112 return make_error("'" + Value + "' not an integer",
113 inconvertibleErrorCode());
114 Policy.MaxSizeBytes = Size * Mult;
94115 } else {
95116 return make_error("Unknown key: '" + Key + "'",
96117 inconvertibleErrorCode());
114135 if (!isPathDir)
115136 return false;
116137
117 Policy.PercentageOfAvailableSpace =
118 std::min(Policy.PercentageOfAvailableSpace, 100u);
138 Policy.MaxSizePercentageOfAvailableSpace =
139 std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
119140
120141 if (Policy.Expiration == seconds(0) &&
121 Policy.PercentageOfAvailableSpace == 0) {
142 Policy.MaxSizePercentageOfAvailableSpace == 0 &&
143 Policy.MaxSizeBytes == 0) {
122144 DEBUG(dbgs() << "No pruning settings set, exit early\n");
123145 // Nothing will be pruned, early exit
124146 return false;
156178 writeTimestampFile(TimestampFile);
157179 }
158180
159 bool ShouldComputeSize = (Policy.PercentageOfAvailableSpace > 0);
181 bool ShouldComputeSize =
182 (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
160183
161184 // Keep track of space
162185 std::set> FileSizes;
215238 }
216239 sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
217240 auto AvailableSpace = TotalSize + SpaceInfo.free;
241
242 if (Policy.MaxSizePercentageOfAvailableSpace == 0)
243 Policy.MaxSizePercentageOfAvailableSpace = 100;
244 if (Policy.MaxSizeBytes == 0)
245 Policy.MaxSizeBytes = AvailableSpace;
246 auto TotalSizeTarget = std::min(
247 AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
248 Policy.MaxSizeBytes);
249
250 DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
251 << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
252 << "%, " << Policy.MaxSizeBytes << " bytes\n");
253
218254 auto FileAndSize = FileSizes.rbegin();
219 DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
220 << "% target is: " << Policy.PercentageOfAvailableSpace
221 << "\n");
222255 // Remove the oldest accessed files first, till we get below the threshold
223 while (((100 * TotalSize) / AvailableSpace) >
224 Policy.PercentageOfAvailableSpace &&
225 FileAndSize != FileSizes.rend()) {
256 while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
226257 // Remove the file.
227258 sys::fs::remove(FileAndSize->second);
228259 // Update size
1717 ASSERT_TRUE(bool(P));
1818 EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
1919 EXPECT_EQ(std::chrono::hours(7 * 24), P->Expiration);
20 EXPECT_EQ(75u, P->PercentageOfAvailableSpace);
20 EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
2121 }
2222
2323 TEST(CachePruningPolicyParser, Interval) {
3838 EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
3939 }
4040
41 TEST(CachePruningPolicyParser, PercentageOfAvailableSpace) {
41 TEST(CachePruningPolicyParser, MaxSizePercentageOfAvailableSpace) {
4242 auto P = parseCachePruningPolicy("cache_size=100%");
4343 ASSERT_TRUE(bool(P));
44 EXPECT_EQ(100u, P->PercentageOfAvailableSpace);
44 EXPECT_EQ(100u, P->MaxSizePercentageOfAvailableSpace);
45 EXPECT_EQ(0u, P->MaxSizeBytes);
46 }
47
48 TEST(CachePruningPolicyParser, MaxSizeBytes) {
49 auto P = parseCachePruningPolicy("cache_size_bytes=1");
50 ASSERT_TRUE(bool(P));
51 EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
52 EXPECT_EQ(1u, P->MaxSizeBytes);
53 P = parseCachePruningPolicy("cache_size_bytes=2k");
54 ASSERT_TRUE(bool(P));
55 EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
56 EXPECT_EQ(2u * 1024u, P->MaxSizeBytes);
57 P = parseCachePruningPolicy("cache_size_bytes=3m");
58 ASSERT_TRUE(bool(P));
59 EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
60 EXPECT_EQ(3u * 1024u * 1024u, P->MaxSizeBytes);
61 P = parseCachePruningPolicy("cache_size_bytes=4g");
62 ASSERT_TRUE(bool(P));
63 EXPECT_EQ(75u, P->MaxSizePercentageOfAvailableSpace);
64 EXPECT_EQ(4ull * 1024ull * 1024ull * 1024ull, P->MaxSizeBytes);
4565 }
4666
4767 TEST(CachePruningPolicyParser, Multiple) {
4969 ASSERT_TRUE(bool(P));
5070 EXPECT_EQ(std::chrono::seconds(1200), P->Interval);
5171 EXPECT_EQ(std::chrono::seconds(1), P->Expiration);
52 EXPECT_EQ(50u, P->PercentageOfAvailableSpace);
72 EXPECT_EQ(50u, P->MaxSizePercentageOfAvailableSpace);
5373 }
5474
5575 TEST(CachePruningPolicyParser, Errors) {
6585 toString(parseCachePruningPolicy("cache_size=foo%").takeError()));
6686 EXPECT_EQ("'101' must be between 0 and 100",
6787 toString(parseCachePruningPolicy("cache_size=101%").takeError()));
88 EXPECT_EQ(
89 "'foo' not an integer",
90 toString(parseCachePruningPolicy("cache_size_bytes=foo").takeError()));
91 EXPECT_EQ(
92 "'foo' not an integer",
93 toString(parseCachePruningPolicy("cache_size_bytes=foom").takeError()));
6894 EXPECT_EQ("Unknown key: 'foo'",
6995 toString(parseCachePruningPolicy("foo=bar").takeError()));
7096 }