llvm.org GIT mirror llvm / 57f557b
[lit] Support sharding testsuites, for parallel execution. Summary: This change equips lit.py with two new options, --num-shards=M and --run-shard=N (set by default from env vars LIT_NUM_SHARDS and LIT_RUN_SHARD). The options must be used together, and N must be in 1..M. Together these options effect only test selection: they partition the testsuite into M equal-sized "shards", then select only the Nth shard. They can be used in a cluster of test machines to achieve a very crude (static) form of parallelism, with minimal configuration work. Reviewers: modocache, ddunbar Reviewed By: ddunbar Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D28789 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@292417 91177308-0d34-0410-b5e6-96231b3b80d8 Graydon Hoare 2 years ago
3 changed file(s) with 138 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
150150 .. option:: --shuffle
151151
152152 Run the tests in a random order.
153
154 .. option:: --num-shards=M
155
156 Divide the set of selected tests into ``M`` equal-sized subsets or
157 "shards", and run only one of them. Must be used with the
158 ``--run-shard=N`` option, which selects the shard to run. The environment
159 variable ``LIT_NUM_SHARDS`` can also be used in place of this
160 option. These two options provide a coarse mechanism for paritioning large
161 testsuites, for parallel execution on separate machines (say in a large
162 testing farm).
163
164 .. option:: --run-shard=N
165
166 Select which shard to run, assuming the ``--num-shards=M`` option was
167 provided. The two options must be used together, and the value of ``N``
168 must be in the range ``1..M``. The environment variable
169 ``LIT_RUN_SHARD`` can also be used in place of this option.
153170
154171 ADDITIONAL OPTIONS
155172 ------------------
258258 help=("Only run tests with paths matching the given "
259259 "regular expression"),
260260 action="store", default=None)
261 selection_group.add_argument("--num-shards", dest="numShards", metavar="M",
262 help="Split testsuite into M pieces and only run one",
263 action="store", type=int,
264 default=os.environ.get("LIT_NUM_SHARDS"))
265 selection_group.add_argument("--run-shard", dest="runShard", metavar="N",
266 help="Run shard #N of the testsuite",
267 action="store", type=int,
268 default=os.environ.get("LIT_RUN_SHARD"))
261269
262270 debug_group = parser.add_argument_group("Debug and Experimental Options")
263271 debug_group.add_argument("--debug",
397405 sort_by_incremental_cache(run)
398406 else:
399407 run.tests.sort(key = lambda t: (not t.isEarlyTest(), t.getFullName()))
408
409 # Then optionally restrict our attention to a shard of the tests.
410 if (opts.numShards is not None) or (opts.runShard is not None):
411 if (opts.numShards is None) or (opts.runShard is None):
412 parser.error("--num-shards and --run-shard must be used together")
413 if opts.numShards <= 0:
414 parser.error("--num-shards must be positive")
415 if (opts.runShard < 1) or (opts.runShard > opts.numShards):
416 parser.error("--run-shard must be between 1 and --num-shards (inclusive)")
417 num_tests = len(run.tests)
418 # Note: user views tests and shard numbers counting from 1.
419 test_ixs = range(opts.runShard - 1, num_tests, opts.numShards)
420 run.tests = [run.tests[i] for i in test_ixs]
421 # Generate a preview of the first few test indices in the shard
422 # to accompany the arithmetic expression, for clarity.
423 preview_len = 3
424 ix_preview = ", ".join([str(i+1) for i in test_ixs[:preview_len]])
425 if len(test_ixs) > preview_len:
426 ix_preview += ", ..."
427 litConfig.note('Selecting shard %d/%d = size %d/%d = tests #(%d*k)+%d = [%s]' %
428 (opts.runShard, opts.numShards,
429 len(run.tests), num_tests,
430 opts.numShards, opts.runShard, ix_preview))
400431
401432 # Finally limit the number of tests, if desired.
402433 if opts.maxTests is not None:
0 # RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s
1 # CHECK-BASIC: Testing: 5 tests
2
3
4 # Check that regex-filtering works
5 #
6 # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
7 # CHECK-FILTER: Testing: 2 of 5 tests
8
9
10 # Check that maximum counts work
11 #
12 # RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
13 # CHECK-MAX: Testing: 3 of 5 tests
14
15
16 # Check that sharding partitions the testsuite in a way that distributes the
17 # rounding error nicely (i.e. 5/3 => 2 2 1, not 1 1 3 or whatever)
18 #
19 # RUN: %{lit} --num-shards 3 --run-shard 1 %{inputs}/discovery >%t.out 2>%t.err
20 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ERR < %t.err %s
21 # RUN: FileCheck --check-prefix=CHECK-SHARD0-OUT < %t.out %s
22 # CHECK-SHARD0-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
23 # CHECK-SHARD0-OUT: Testing: 2 of 5 tests
24 #
25 # RUN: %{lit} --num-shards 3 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
26 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ERR < %t.err %s
27 # RUN: FileCheck --check-prefix=CHECK-SHARD1-OUT < %t.out %s
28 # CHECK-SHARD1-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
29 # CHECK-SHARD1-OUT: Testing: 2 of 5 tests
30 #
31 # RUN: %{lit} --num-shards 3 --run-shard 3 %{inputs}/discovery >%t.out 2>%t.err
32 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ERR < %t.err %s
33 # RUN: FileCheck --check-prefix=CHECK-SHARD2-OUT < %t.out %s
34 # CHECK-SHARD2-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
35 # CHECK-SHARD2-OUT: Testing: 1 of 5 tests
36
37
38 # Check that sharding via env vars works.
39 #
40 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=1 %{lit} %{inputs}/discovery >%t.out 2>%t.err
41 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-ERR < %t.err %s
42 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-OUT < %t.out %s
43 # CHECK-SHARD0-ENV-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
44 # CHECK-SHARD0-ENV-OUT: Testing: 2 of 5 tests
45 #
46 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=2 %{lit} %{inputs}/discovery >%t.out 2>%t.err
47 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-ERR < %t.err %s
48 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-OUT < %t.out %s
49 # CHECK-SHARD1-ENV-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
50 # CHECK-SHARD1-ENV-OUT: Testing: 2 of 5 tests
51 #
52 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=3 %{lit} %{inputs}/discovery >%t.out 2>%t.err
53 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-ERR < %t.err %s
54 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-OUT < %t.out %s
55 # CHECK-SHARD2-ENV-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
56 # CHECK-SHARD2-ENV-OUT: Testing: 1 of 5 tests
57
58
59 # Check that providing more shards than tests results in 1 test per shard
60 # until we run out, then 0.
61 #
62 # RUN: %{lit} --num-shards 100 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
63 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR1 < %t.err %s
64 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT1 < %t.out %s
65 # CHECK-SHARD-BIG-ERR1: note: Selecting shard 2/100 = size 1/5 = tests #(100*k)+2 = [2]
66 # CHECK-SHARD-BIG-OUT1: Testing: 1 of 5 tests
67 #
68 # RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err
69 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s
70 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s
71 # CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []
72 # CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests
73 #
74 # RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err
75 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s
76 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s
77 # CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []
78 # CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests
79
80
81 # Check that range constraints are enforced
82 #
83 # RUN: not %{lit} --num-shards 0 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
84 # RUN: FileCheck --check-prefix=CHECK-SHARD-ERR < %t.err %s
85 # CHECK-SHARD-ERR: error: --num-shards must be positive
86 #
87 # RUN: not %{lit} --num-shards 3 --run-shard 4 %{inputs}/discovery >%t.out 2>%t.err
88 # RUN: FileCheck --check-prefix=CHECK-SHARD-ERR2 < %t.err %s
89 # CHECK-SHARD-ERR2: error: --run-shard must be between 1 and --num-shards (inclusive)