llvm.org GIT mirror llvm / e2a5365
[lit] Enqueue tests on a separate thread to not hit limits on parallel queues Summary: The multiprocessing.Queue.put() call can hang if we try queueing all the tests before starting to take them out of the queue. The current implementation hangs if tests exceed 2^^15, on Mac OS X. This might happen with a ninja check-all if one has a bunch of llvm projects. Reviewers: delcypher, bkramer Subscribers: llvm-commits Differential Revision: http://reviews.llvm.org/D17609 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@263731 91177308-0d34-0410-b5e6-96231b3b80d8 Filipe Cabecinhas 3 years ago
1 changed file(s) with 14 addition(s) and 2 deletion(s). Raw diff Collapse all Expand all
4343 value = property(_get_value, _set_value)
4444
4545 class TestProvider(object):
46 def __init__(self, tests, num_jobs, queue_impl, canceled_flag):
46 def __init__(self, queue_impl, canceled_flag):
4747 self.canceled_flag = canceled_flag
4848
4949 # Create a shared queue to provide the test indices.
5050 self.queue = queue_impl()
51
52 def queue_tests(self, tests, num_jobs):
5153 for i in range(len(tests)):
5254 self.queue.put(i)
5355 for i in range(num_jobs):
228230 consumer = ThreadResultsConsumer(display)
229231
230232 # Create the test provider.
231 provider = TestProvider(self.tests, jobs, queue_impl, canceled_flag)
233 provider = TestProvider(queue_impl, canceled_flag)
234
235 # Queue the tests outside the main thread because we can't guarantee
236 # that we can put() all the tests without blocking:
237 # https://docs.python.org/2/library/multiprocessing.html
238 # e.g: On Mac OS X, we will hang if we put 2^15 elements in the queue
239 # without taking any out.
240 queuer = task_impl(target=provider.queue_tests, args=(self.tests, jobs))
241 queuer.start()
232242
233243 # Install a console-control signal handler on Windows.
234244 if win32api is not None:
250260 else:
251261 # Otherwise, execute the tests in parallel
252262 self._execute_tests_in_parallel(task_impl, provider, consumer, jobs)
263
264 queuer.join()
253265
254266 # Cancel the timeout handler.
255267 if max_time is not None: