llvm.org GIT mirror llvm / 1d95844
[lit] Report line number for failed RUN command (Relands r333584, reverted in 333592.) When debugging test failures with -vv (or -v in the case of the internal shell), this makes it easier to locate the RUN line that failed. For example, clang's test/Driver/linux-ld.c has 892 total RUN lines, and clang's test/Driver/arm-cortex-cpus.c has 424 RUN lines after concatenation for line continuations. When reading the generated shell script, this also makes it easier to locate the RUN line that produced each command. To support reporting RUN line numbers in the case of the internal shell, this patch extends the internal shell to support the null command, ":", except pipelines are not supported. To support reporting RUN line numbers in the case of windows cmd.exe as the external shell, this patch extends -vv to set "echo on" instead of "echo off" in bat files. (Support for windows cmd.exe as a lit external shell will likely be dropped later, but I found out too late.) Reviewed By: delcypher, asmith, stella.stamenova, jmorse, lebedev.ri, rnk Differential Revision: https://reviews.llvm.org/D44598 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@333614 91177308-0d34-0410-b5e6-96231b3b80d8 Joel E. Denny 1 year, 4 months ago
17 changed file(s) with 185 addition(s) and 11 deletion(s). Raw diff Collapse all Expand all
8484 Echo all commands to stdout, as they are being executed.
8585 This can be valuable for debugging test failures, as the last echoed command
8686 will be the one which has failed.
87 :program:`lit` normally inserts a no-op command (``:`` in the case of bash)
88 with argument ``'RUN: at line N'`` before each command pipeline, and this
89 option also causes those no-op commands to be echoed to stdout to help you
90 locate the source line of the failed command.
8791 This option implies ``--verbose``.
8892
8993 .. option:: -a, --show-all
3838 # Use temporary files to replace /dev/null on Windows.
3939 kAvoidDevNull = kIsWindows
4040 kDevNull = "/dev/null"
41
42 # A regex that matches %dbg(ARG), which lit inserts at the beginning of each
43 # run command pipeline such that ARG specifies the pipeline's source line
44 # number. lit later expands each %dbg(ARG) to a command that behaves as a null
45 # command in the target shell so that the line number is seen in lit's verbose
46 # mode.
47 #
48 # This regex captures ARG. ARG must not contain a right parenthesis, which
49 # terminates %dbg. ARG must not contain quotes, in which ARG might be enclosed
50 # during expansion.
51 kPdbgRegex = '%dbg\(([^)\'"]*)\)'
4152
4253 class ShellEnvironment(object):
4354
788799 results.append(cmdResult)
789800 return cmdResult.exitCode
790801
802 if cmd.commands[0].args[0] == ':':
803 if len(cmd.commands) != 1:
804 raise InternalShellError(cmd.commands[0], "Unsupported: ':' "
805 "cannot be part of a pipeline")
806 results.append(ShellCommandResult(cmd.commands[0], '', '', 0, False))
807 return 0;
808
791809 procs = []
792810 default_stdin = subprocess.PIPE
793811 stderrTempFiles = []
981999
9821000 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
9831001 cmds = []
984 for ln in commands:
1002 for i, ln in enumerate(commands):
1003 ln = commands[i] = re.sub(kPdbgRegex, ": '\\1'", ln)
9851004 try:
9861005 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
9871006 test.config.pipefail).parse())
10651084 mode += 'b' # Avoid CRLFs when writing bash scripts.
10661085 f = open(script, mode)
10671086 if isWin32CMDEXE:
1068 f.write('@echo off\n')
1069 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
1087 for i, ln in enumerate(commands):
1088 commands[i] = re.sub(kPdbgRegex, "echo '\\1' > nul", ln)
1089 if litConfig.echo_all_commands:
1090 f.write('@echo on\n')
1091 else:
1092 f.write('@echo off\n')
1093 f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
10701094 else:
1095 for i, ln in enumerate(commands):
1096 commands[i] = re.sub(kPdbgRegex, ": '\\1'", ln)
10711097 if test.config.pipefail:
10721098 f.write('set -o pipefail;')
10731099 if litConfig.echo_all_commands:
13001326 self.parser = parser
13011327
13021328 if kind == ParserKind.COMMAND:
1303 self.parser = self._handleCommand
1329 self.parser = lambda line_number, line, output: \
1330 self._handleCommand(line_number, line, output,
1331 self.keyword)
13041332 elif kind == ParserKind.LIST:
13051333 self.parser = self._handleList
13061334 elif kind == ParserKind.BOOLEAN_EXPR:
13311359 return (not line.strip() or output)
13321360
13331361 @staticmethod
1334 def _handleCommand(line_number, line, output):
1362 def _handleCommand(line_number, line, output, keyword):
13351363 """A helper for parsing COMMAND type keywords"""
13361364 # Trim trailing whitespace.
13371365 line = line.rstrip()
13501378 else:
13511379 if output is None:
13521380 output = []
1381 pdbg = "%dbg({keyword} at line {line_number})".format(
1382 keyword=keyword,
1383 line_number=line_number)
1384 assert re.match(kPdbgRegex + "$", pdbg), \
1385 "kPdbgRegex expected to match actual %dbg usage"
1386 line = "{pdbg} && {real_command}".format(
1387 pdbg=pdbg,
1388 real_command=line)
13531389 output.append(line)
13541390 return output
13551391
0 # These commands must run under both bash and windows cmd.exe (with GnuWin32
1 # tools).
2
3 # RUN: true
4 # RUN: false
5 # RUN: true
0 # These commands must run under both bash and windows cmd.exe (with GnuWin32
1 # tools).
2
3 # RUN: echo 'foo bar' \
4 # RUN: | FileCheck %s
5 # RUN: echo \
6 # RUN: 'foo baz' \
7 # RUN: | FileCheck %s
8 # RUN: echo 'foo bar' \
9 # RUN: | FileCheck %s
10
11 # CHECK: foo bar
0 import lit.formats
1 config.test_format = lit.formats.ShTest(execute_external=True)
0 # RUN: : first line continued \
1 # RUN: to second line
2 # RUN: echo 'foo bar' \
3 # RUN: | FileCheck %s
4 # RUN: echo \
5 # RUN: 'foo baz' \
6 # RUN: | FileCheck %s
7 # RUN: echo 'foo bar' \
8 # RUN: | FileCheck %s
9
10 # CHECK: foo bar
0 import lit.formats
1 config.test_format = lit.formats.ShTest(execute_external=False)
0 config.name = 'shtest-run-at-line'
1 config.suffixes = ['.txt']
0 # Check error on an unsupported ":". (cannot be part of a pipeline)
1 #
2 # RUN: : | echo "hello"
7070 if directory:
7171 path = os.path.pathsep.join((directory, path))
7272 config.environment['PATH'] = path
73
74 # These substitutions are needed only in tests where the external shell is used
75 # and could be either bash or windows cmd.exe. Substitutions are expected to
76 # be expanded in double quotes.
77 isWin32CMDEXE = lit_config.isWindows and not lit_config.getBashPath()
78 if isWin32CMDEXE:
79 config.substitutions.append(('%{pdbg0}', "echo '"))
80 config.substitutions.append(('%{pdbg1}', "' > nul"))
81 else:
82 config.substitutions.append(('%{pdbg0}', ": '"))
83 config.substitutions.append(('%{pdbg1}', "'"))
77 #
88 # END.
99
10 # CHECK: Failing Tests (26)
10 # CHECK: Failing Tests (27)
1111 # CHECK: Failing Tests (1)
1212 # CHECK: Failing Tests (2)
1313 # CHECK: error: Setting --max-failures to 0 does not have any effect.
3838 #
3939 # CHECK: Command Output (stdout):
4040 # CHECK-NEXT: --
41 # CHECK-NEXT: $ ":" "RUN: at line 1"
4142 # CHECK-NEXT: $ "printf"
4243 # CHECK-NEXT: # command output:
4344 # CHECK-NEXT: line 1: failed test output on stdout
1515 #
1616 # CHECK: Command Output
1717 # CHECK-NEXT: --
18 # CHECK-NEXT: $ ":" "RUN: at line 1"
1819 # CHECK-NEXT: $ "true"
20 # CHECK-NEXT: $ ":" "RUN: at line 2"
1921 # CHECK-NEXT: $ "echo" "hi"
2022 # CHECK-NEXT: # command output:
2123 # CHECK-NEXT: hi
2224 #
23 # CHECK: $ "wc" "missing-file"
25 # CHECK: $ ":" "RUN: at line 3"
26 # CHECK-NEXT: $ "wc" "missing-file"
2427 # CHECK-NEXT: # redirected output from '{{.*(/|\\\\)}}basic.txt.tmp.out':
2528 # CHECK-NEXT: missing-file{{.*}} No such file or directory
2629 # CHECK: note: command had no output on stdout or stderr
0 # Check that -vv makes the line number of the failing RUN command clear.
1 # (-v is actually sufficient in the case of the internal shell.)
2 #
3 # RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
4 # RUN: FileCheck --input-file %t.out -Dpdbg0="%{pdbg0}" -Dpdbg1="%{pdbg1}" %s
5 #
6 # END.
7
8
9 # CHECK: Testing: 4 tests
10
11
12 # In the case of the external shell, we check for only RUN lines in stderr in
13 # case some shell implementations format "set -x" output differently.
14
15 # CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
16
17 # CHECK: Script:
18 # CHECK: [[pdbg0]]RUN: at line 4[[pdbg1]] && true
19 # CHECK-NEXT: [[pdbg0]]RUN: at line 5[[pdbg1]] && false
20 # CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] && true
21
22 # CHECK: RUN: at line 4
23 # CHECK: RUN: at line 5
24 # CHECK-NOT: RUN
25
26 # CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
27
28 # CHECK: Script:
29 # CHECK: [[pdbg0]]RUN: at line 4[[pdbg1]] && echo 'foo bar' | FileCheck
30 # CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] && echo 'foo baz' | FileCheck
31 # CHECK-NEXT: [[pdbg0]]RUN: at line 9[[pdbg1]] && echo 'foo bar' | FileCheck
32
33 # CHECK: RUN: at line 4
34 # CHECK: RUN: at line 6
35 # CHECK-NOT: RUN
36
37
38 # CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
39
40 # CHECK: Script:
41 # CHECK: : 'RUN: at line 1' && true
42 # CHECK-NEXT: : 'RUN: at line 2' && false
43 # CHECK-NEXT: : 'RUN: at line 3' && true
44
45 # CHECK: Command Output (stdout)
46 # CHECK: $ ":" "RUN: at line 1"
47 # CHECK-NEXT: $ "true"
48 # CHECK-NEXT: $ ":" "RUN: at line 2"
49 # CHECK-NEXT: $ "false"
50 # CHECK-NOT: RUN
51
52 # CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
53
54 # CHECK: Script:
55 # CHECK: : 'RUN: at line 1' && : first line continued to second line
56 # CHECK-NEXT: : 'RUN: at line 3' && echo 'foo bar' | FileCheck
57 # CHECK-NEXT: : 'RUN: at line 5' && echo 'foo baz' | FileCheck
58 # CHECK-NEXT: : 'RUN: at line 8' && echo 'foo bar' | FileCheck
59
60 # CHECK: Command Output (stdout)
61 # CHECK: $ ":" "RUN: at line 1"
62 # CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
63 # CHECK-NEXT: $ ":" "RUN: at line 3"
64 # CHECK-NEXT: $ "echo" "foo bar"
65 # CHECK-NEXT: $ "FileCheck" "{{.*}}"
66 # CHECK-NEXT: $ ":" "RUN: at line 5"
67 # CHECK-NEXT: $ "echo" "foo baz"
68 # CHECK-NEXT: $ "FileCheck" "{{.*}}"
69 # CHECK-NOT: RUN
2525 # CHECK: error: command failed with exit status: 1
2626 # CHECK: ***
2727
28 # CHECK: FAIL: shtest-shell :: colon-error.txt
29 # CHECK: *** TEST 'shtest-shell :: colon-error.txt' FAILED ***
30 # CHECK: $ ":"
31 # CHECK: # command stderr:
32 # CHECK: Unsupported: ':' cannot be part of a pipeline
33 # CHECK: error: command failed with exit status: 127
34 # CHECK: ***
35
2836 # CHECK: FAIL: shtest-shell :: diff-error-0.txt
2937 # CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
3038 # CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
152160 #
153161 # CHECK: FAIL: shtest-shell :: error-1.txt
154162 # CHECK: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
155 # CHECK: shell parser error on: 'echo "missing quote'
163 # CHECK: shell parser error on: ': \'RUN: at line 3\' && echo "missing quote'
156164 # CHECK: ***
157165
158166 # CHECK: FAIL: shtest-shell :: error-2.txt
218226 # CHECK: PASS: shtest-shell :: sequencing-0.txt
219227 # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
220228 # CHECK: PASS: shtest-shell :: valid-shell.txt
221 # CHECK: Failing Tests (26)
229 # CHECK: Failing Tests (27)
9898 cmd_parser = self.get_parser(parsers, 'MY_RUN:')
9999 value = cmd_parser.getValue()
100100 self.assertEqual(len(value), 2) # there are only two run lines
101 self.assertEqual(value[0].strip(), 'baz')
102 self.assertEqual(value[1].strip(), 'foo bar')
101 self.assertEqual(value[0].strip(), "%dbg(MY_RUN: at line 4) && baz")
102 self.assertEqual(value[1].strip(), "%dbg(MY_RUN: at line 7) && foo bar")
103103
104104 def test_custom(self):
105105 parsers = self.make_parsers()