llvm.org GIT mirror llvm / 19c9b1c
[utils] Refactor utils/update_{,llc_}test_checks.py to share more code Summary: This revision refactors 1. parser 2. CHECK line adder of utils/update_{,llc_}test_checks.py so that thir functionality can be re-used by other utility scripts (e.g. D42712) Reviewers: asb, craig.topper, RKSimon, echristo Subscribers: llvm-commits, spatel Differential Revision: https://reviews.llvm.org/D42805 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@324803 91177308-0d34-0410-b5e6-96231b3b80d8 Fangrui Song 2 years ago
4 changed file(s) with 190 addition(s) and 166 deletion(s). Raw diff Collapse all Expand all
0 import re
1 import string
1 import sys
22
33 from . import common
44
5 if sys.version_info[0] > 2:
6 class string:
7 expandtabs = str.expandtabs
8 else:
9 import string
10
511 # RegEx: this is where the magic happens.
12
13 ##### Assembly parser
614
715 ASM_FUNCTION_X86_RE = re.compile(
816 r'^_?(?P[^:]+):[ \t]*#+[ \t]*@(?P=func)\n[^:]*?'
196204 common.build_function_body_dictionary(
197205 function_re, scrubber, [args], raw_tool_output, prefixes,
198206 func_dict, args.verbose)
207
208 ##### Generator of assembly CHECK lines
209
210 def add_asm_checks(output_lines, comment_marker, run_list, func_dict, func_name):
211 printed_prefixes = []
212 for p in run_list:
213 checkprefixes = p[0]
214 for checkprefix in checkprefixes:
215 if checkprefix in printed_prefixes:
216 break
217 # TODO func_dict[checkprefix] may be None, '' or not exist.
218 # Fix the call sites.
219 if func_name not in func_dict[checkprefix] or not func_dict[checkprefix][func_name]:
220 continue
221 # Add some space between different check prefixes.
222 if len(printed_prefixes) != 0:
223 output_lines.append(comment_marker)
224 printed_prefixes.append(checkprefix)
225 output_lines.append('%s %s-LABEL: %s:' % (comment_marker, checkprefix, func_name))
226 func_body = func_dict[checkprefix][func_name].splitlines()
227 output_lines.append('%s %s: %s' % (comment_marker, checkprefix, func_body[0]))
228 for func_line in func_body[1:]:
229 output_lines.append('%s %s-NEXT: %s' % (comment_marker, checkprefix, func_line))
230 # Add space between different check prefixes and the first line of code.
231 # output_lines.append(';')
232 break
0 from __future__ import print_function
11 import re
2 import string
23 import subprocess
34 import sys
45
5 RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$')
6 CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?=(\S+)')
7 CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
8
9 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@(\w+)\s*\(')
10 TRIPLE_IR_RE = re.compile(r'^target\s+triple\s*=\s*"([^"]+)"$')
11 TRIPLE_ARG_RE = re.compile(r'-mtriple=([^ ]+)')
12
13 SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)')
14 SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M)
15 SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M)
16 SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n')
17 SCRUB_LOOP_COMMENT_RE = re.compile(
18 r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M)
6 if sys.version_info[0] > 2:
7 class string:
8 expandtabs = str.expandtabs
9 else:
10 import string
11
12 ##### Common utilities for update_*test_checks.py
1913
2014 def should_add_line_to_output(input_line, prefix_set):
2115 # Skip any blank comment lines in the IR.
4034 stdout = stdout.decode()
4135 # Fix line endings to unix CR style.
4236 return stdout.replace('\r\n', '\n')
37
38 ##### LLVM IR parser
39
40 RUN_LINE_RE = re.compile('^\s*;\s*RUN:\s*(.*)$')
41 CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?=(\S+)')
42 CHECK_RE = re.compile(r'^\s*;\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
43
44 OPT_FUNCTION_RE = re.compile(
45 r'^\s*define\s+(?:internal\s+)?[^@]*@(?P[\w-]+?)\s*\('
46 r'(\s+)?[^)]*[^{]*\{\n(?P.*?)^\}$',
47 flags=(re.M | re.S))
48
49 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@(\w+)\s*\(')
50 TRIPLE_IR_RE = re.compile(r'^target\s+triple\s*=\s*"([^"]+)"$')
51 TRIPLE_ARG_RE = re.compile(r'-mtriple=([^ ]+)')
52
53 SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)')
54 SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M)
55 SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M)
56 SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n')
57 SCRUB_LOOP_COMMENT_RE = re.compile(
58 r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M)
59
60 def scrub_body(body):
61 # Scrub runs of whitespace out of the assembly, but leave the leading
62 # whitespace in place.
63 body = SCRUB_WHITESPACE_RE.sub(r' ', body)
64 # Expand the tabs used for indentation.
65 body = string.expandtabs(body, 2)
66 # Strip trailing whitespace.
67 body = SCRUB_TRAILING_WHITESPACE_RE.sub(r'', body)
68 return body
4369
4470 # Build up a dictionary of all the function bodies.
4571 def build_function_body_dictionary(function_re, scrubber, scrubber_args, raw_tool_output, prefixes, func_dict, verbose):
6591 continue
6692
6793 func_dict[prefix][func] = scrubbed_body
94
95 ##### Generator of LLVM IR CHECK lines
96
97 SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*')
98
99 # Match things that look at identifiers, but only if they are followed by
100 # spaces, commas, paren, or end of the string
101 IR_VALUE_RE = re.compile(r'(\s+)%([\w\.]+?)([,\s\(\)]|\Z)')
102
103 # Create a FileCheck variable name based on an IR name.
104 def get_value_name(var):
105 if var.isdigit():
106 var = 'TMP' + var
107 var = var.replace('.', '_')
108 return var.upper()
109
110
111 # Create a FileCheck variable from regex.
112 def get_value_definition(var):
113 return '[[' + get_value_name(var) + ':%.*]]'
114
115
116 # Use a FileCheck variable.
117 def get_value_use(var):
118 return '[[' + get_value_name(var) + ']]'
119
120 # Replace IR value defs and uses with FileCheck variables.
121 def genericize_check_lines(lines):
122 # This gets called for each match that occurs in
123 # a line. We transform variables we haven't seen
124 # into defs, and variables we have seen into uses.
125 def transform_line_vars(match):
126 var = match.group(2)
127 if var in vars_seen:
128 rv = get_value_use(var)
129 else:
130 vars_seen.add(var)
131 rv = get_value_definition(var)
132 # re.sub replaces the entire regex match
133 # with whatever you return, so we have
134 # to make sure to hand it back everything
135 # including the commas and spaces.
136 return match.group(1) + rv + match.group(3)
137
138 vars_seen = set()
139 lines_with_def = []
140
141 for i, line in enumerate(lines):
142 # An IR variable named '%.' matches the FileCheck regex string.
143 line = line.replace('%.', '%dot')
144 # Ignore any comments, since the check lines will too.
145 scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r'', line)
146 lines[i] = IR_VALUE_RE.sub(transform_line_vars, scrubbed_line)
147 return lines
148
149
150 def add_ir_checks(output_lines, prefix_list, func_dict, func_name, opt_basename):
151 # Label format is based on IR string.
152 check_label_format = "; %s-LABEL: @%s("
153
154 printed_prefixes = []
155 for checkprefixes, _ in prefix_list:
156 for checkprefix in checkprefixes:
157 if checkprefix in printed_prefixes:
158 break
159 if not func_dict[checkprefix][func_name]:
160 continue
161 # Add some space between different check prefixes, but not after the last
162 # check line (before the test code).
163 #if len(printed_prefixes) != 0:
164 # output_lines.append(';')
165 printed_prefixes.append(checkprefix)
166 output_lines.append(check_label_format % (checkprefix, func_name))
167 func_body = func_dict[checkprefix][func_name].splitlines()
168
169 # For IR output, change all defs to FileCheck variables, so we're immune
170 # to variable naming fashions.
171 func_body = genericize_check_lines(func_body)
172
173 # This could be selectively enabled with an optional invocation argument.
174 # Disabled for now: better to check everything. Be safe rather than sorry.
175
176 # Handle the first line of the function body as a special case because
177 # it's often just noise (a useless asm comment or entry label).
178 #if func_body[0].startswith("#") or func_body[0].startswith("entry:"):
179 # is_blank_line = True
180 #else:
181 # output_lines.append('; %s: %s' % (checkprefix, func_body[0]))
182 # is_blank_line = False
183
184 is_blank_line = False
185
186 for func_line in func_body:
187 if func_line.strip() == '':
188 is_blank_line = True
189 continue
190 # Do not waste time checking IR comments.
191 func_line = SCRUB_IR_COMMENT_RE.sub(r'', func_line)
192
193 # Skip blank lines instead of checking them.
194 if is_blank_line == True:
195 output_lines.append('; %s: %s' % (checkprefix, func_line))
196 else:
197 output_lines.append('; %s-NEXT: %s' % (checkprefix, func_line))
198 is_blank_line = False
199
200 # Add space between different check prefixes and also before the first
201 # line of code in the test function.
202 output_lines.append(';')
203 break
204 return output_lines
1616 from UpdateTestChecks import asm, common
1717
1818 ADVERT = '; NOTE: Assertions have been autogenerated by '
19
20
21 def add_checks(output_lines, run_list, func_dict, func_name):
22 printed_prefixes = []
23 for p in run_list:
24 checkprefixes = p[0]
25 for checkprefix in checkprefixes:
26 if checkprefix in printed_prefixes:
27 break
28 if not func_dict[checkprefix][func_name]:
29 continue
30 # Add some space between different check prefixes.
31 if len(printed_prefixes) != 0:
32 output_lines.append(';')
33 printed_prefixes.append(checkprefix)
34 output_lines.append('; %s-LABEL: %s:' % (checkprefix, func_name))
35 func_body = func_dict[checkprefix][func_name].splitlines()
36 output_lines.append('; %s: %s' % (checkprefix, func_body[0]))
37 for func_line in func_body[1:]:
38 output_lines.append('; %s-NEXT: %s' % (checkprefix, func_line))
39 # Add space between different check prefixes and the first line of code.
40 # output_lines.append(';')
41 break
42 return output_lines
4319
4420
4521 def main():
155131 continue
156132
157133 # Print out the various check lines here.
158 output_lines = add_checks(output_lines, run_list, func_dict, func_name)
134 asm.add_asm_checks(output_lines, ';', run_list, func_dict, func_name)
159135 is_in_function_start = False
160136
161137 if is_in_function:
4343
4444 # RegEx: this is where the magic happens.
4545
46 SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*')
47
4846 IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
49 OPT_FUNCTION_RE = re.compile(
50 r'^\s*define\s+(?:internal\s+)?[^@]*@(?P[\w-]+?)\s*\('
51 r'(\s+)?[^)]*[^{]*\{\n(?P.*?)^\}$',
52 flags=(re.M | re.S))
53 # Match things that look at identifiers, but only if they are followed by
54 # spaces, commas, paren, or end of the string
55 IR_VALUE_RE = re.compile(r'(\s+)%([\w\.]+?)([,\s\(\)]|\Z)')
5647
5748
5849
59 def scrub_body(body, opt_basename):
60 # Scrub runs of whitespace out of the assembly, but leave the leading
61 # whitespace in place.
62 body = common.SCRUB_WHITESPACE_RE.sub(r' ', body)
63 # Expand the tabs used for indentation.
64 body = string.expandtabs(body, 2)
65 # Strip trailing whitespace.
66 body = common.SCRUB_TRAILING_WHITESPACE_RE.sub(r'', body)
67 return body
68
69
70
71 # Create a FileCheck variable name based on an IR name.
72 def get_value_name(var):
73 if var.isdigit():
74 var = 'TMP' + var
75 var = var.replace('.', '_')
76 return var.upper()
77
78
79 # Create a FileCheck variable from regex.
80 def get_value_definition(var):
81 return '[[' + get_value_name(var) + ':%.*]]'
82
83
84 # Use a FileCheck variable.
85 def get_value_use(var):
86 return '[[' + get_value_name(var) + ']]'
87
88 # Replace IR value defs and uses with FileCheck variables.
89 def genericize_check_lines(lines):
90 # This gets called for each match that occurs in
91 # a line. We transform variables we haven't seen
92 # into defs, and variables we have seen into uses.
93 def transform_line_vars(match):
94 var = match.group(2)
95 if var in vars_seen:
96 rv = get_value_use(var)
97 else:
98 vars_seen.add(var)
99 rv = get_value_definition(var)
100 # re.sub replaces the entire regex match
101 # with whatever you return, so we have
102 # to make sure to hand it back everything
103 # including the commas and spaces.
104 return match.group(1) + rv + match.group(3)
105
106 vars_seen = set()
107 lines_with_def = []
108
109 for i, line in enumerate(lines):
110 # An IR variable named '%.' matches the FileCheck regex string.
111 line = line.replace('%.', '%dot')
112 # Ignore any comments, since the check lines will too.
113 scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r'', line)
114 lines[i] = IR_VALUE_RE.sub(transform_line_vars, scrubbed_line)
115 return lines
116
117
118 def add_checks(output_lines, prefix_list, func_dict, func_name, opt_basename):
119 # Label format is based on IR string.
120 check_label_format = "; %s-LABEL: @%s("
121
122 printed_prefixes = []
123 for checkprefixes, _ in prefix_list:
124 for checkprefix in checkprefixes:
125 if checkprefix in printed_prefixes:
126 break
127 if not func_dict[checkprefix][func_name]:
128 continue
129 # Add some space between different check prefixes, but not after the last
130 # check line (before the test code).
131 #if len(printed_prefixes) != 0:
132 # output_lines.append(';')
133 printed_prefixes.append(checkprefix)
134 output_lines.append(check_label_format % (checkprefix, func_name))
135 func_body = func_dict[checkprefix][func_name].splitlines()
136
137 # For IR output, change all defs to FileCheck variables, so we're immune
138 # to variable naming fashions.
139 func_body = genericize_check_lines(func_body)
140
141 # This could be selectively enabled with an optional invocation argument.
142 # Disabled for now: better to check everything. Be safe rather than sorry.
143
144 # Handle the first line of the function body as a special case because
145 # it's often just noise (a useless asm comment or entry label).
146 #if func_body[0].startswith("#") or func_body[0].startswith("entry:"):
147 # is_blank_line = True
148 #else:
149 # output_lines.append('; %s: %s' % (checkprefix, func_body[0]))
150 # is_blank_line = False
151
152 is_blank_line = False
153
154 for func_line in func_body:
155 if func_line.strip() == '':
156 is_blank_line = True
157 continue
158 # Do not waste time checking IR comments.
159 func_line = SCRUB_IR_COMMENT_RE.sub(r'', func_line)
160
161 # Skip blank lines instead of checking them.
162 if is_blank_line == True:
163 output_lines.append('; %s: %s' % (checkprefix, func_line))
164 else:
165 output_lines.append('; %s-NEXT: %s' % (checkprefix, func_line))
166 is_blank_line = False
167
168 # Add space between different check prefixes and also before the first
169 # line of code in the test function.
170 output_lines.append(';')
171 break
172 return output_lines
17350
17451
17552 def main():
246123
247124 raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
248125 common.build_function_body_dictionary(
249 OPT_FUNCTION_RE, scrub_body, [opt_basename], raw_tool_output,
250 prefixes, func_dict, args.verbose)
126 common.OPT_FUNCTION_RE, common.scrub_body, [],
127 raw_tool_output, prefixes, func_dict, args.verbose)
251128
252129 is_in_function = False
253130 is_in_function_start = False
268145 continue
269146
270147 # Print out the various check lines here.
271 output_lines = add_checks(output_lines, prefix_list, func_dict, func_name, opt_basename)
148 output_lines = common.add_ir_checks(output_lines, prefix_list, func_dict, func_name, opt_basename)
272149 is_in_function_start = False
273150
274151 if is_in_function: