llvm.org GIT mirror llvm / 9d2ff10
Resubmit "[lit] Refactor out some more common lit configuration code." There were two issues, one Python 3 specific related to Unicode, and another which is that the tool substitution for lld no longer rejected matches where a / preceded the tool name. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@313928 91177308-0d34-0410-b5e6-96231b3b80d8 Zachary Turner 2 years ago
4 changed file(s) with 175 addition(s) and 113 deletion(s). Raw diff Collapse all Expand all
1010 import lit.util
1111 import lit.formats
1212 from lit.llvm import llvm_config
13 from lit.llvm import ToolFilter
1314
1415 # name: The name of this test suite.
1516 config.name = 'LLVM'
133134 # The regex is a pre-assertion to avoid matching a preceding
134135 # dot, hyphen, carat, or slash (.foo, -foo, etc.). Some patterns
135136 # also have a post-assertion to not match a trailing hyphen (foo-).
136 NOJUNK = r"(?
137
138
139 def find_tool_substitution(pattern):
140 # Extract the tool name from the pattern. This relies on the tool
141 # name being surrounded by \b word match operators. If the
142 # pattern starts with "| ", include it in the string to be
143 # substituted.
144 tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
145 pattern)
146 tool_pipe = tool_match.group(2)
147 tool_name = tool_match.group(4)
148 # Did the user specify the tool path + arguments? This allows things like
149 # llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
150 tool_path = lit_config.params.get(tool_name)
151 if tool_path is None:
152 tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
153 if tool_path is None:
154 return tool_name, tool_path, tool_pipe
155 if (tool_name == "llc" and
156 'LLVM_ENABLE_MACHINE_VERIFIER' in os.environ and
157 os.environ['LLVM_ENABLE_MACHINE_VERIFIER'] == "1"):
158 tool_path += " -verify-machineinstrs"
159 if (tool_name == "llvm-go"):
160 tool_path += " go=" + config.go_executable
161 return tool_name, tool_path, tool_pipe
162
163
164 for pattern in [r"\bbugpoint\b(?!-)",
165 NOJUNK + r"\bllc\b",
166 r"\blli\b",
167 r"\bllvm-ar\b",
168 r"\bllvm-as\b",
169 r"\bllvm-bcanalyzer\b",
170 r"\bllvm-config\b",
171 r"\bllvm-cov\b",
172 r"\bllvm-cxxdump\b",
173 r"\bllvm-cvtres\b",
174 r"\bllvm-diff\b",
175 r"\bllvm-dis\b",
176 r"\bllvm-dsymutil\b",
177 r"\bllvm-dwarfdump\b",
178 r"\bllvm-extract\b",
179 r"\bllvm-isel-fuzzer\b",
180 r"\bllvm-lib\b",
181 r"\bllvm-link\b",
182 r"\bllvm-lto\b",
183 r"\bllvm-lto2\b",
184 r"\bllvm-mc\b",
185 r"\bllvm-mcmarkup\b",
186 r"\bllvm-modextract\b",
187 r"\bllvm-nm\b",
188 r"\bllvm-objcopy\b",
189 r"\bllvm-objdump\b",
190 r"\bllvm-pdbutil\b",
191 r"\bllvm-profdata\b",
192 r"\bllvm-ranlib\b",
193 r"\bllvm-readobj\b",
194 r"\bllvm-rtdyld\b",
195 r"\bllvm-size\b",
196 r"\bllvm-split\b",
197 r"\bllvm-strings\b",
198 r"\bllvm-tblgen\b",
199 r"\bllvm-c-test\b",
200 r"\bllvm-cxxfilt\b",
201 r"\bllvm-xray\b",
202 NOJUNK + r"\bllvm-symbolizer\b",
203 NOJUNK + r"\bopt\b",
204 r"\bFileCheck\b",
205 r"\bobj2yaml\b",
206 NOJUNK + r"\bsancov\b",
207 NOJUNK + r"\bsanstats\b",
208 r"\byaml2obj\b",
209 r"\byaml-bench\b",
210 r"\bverify-uselistorder\b",
211 # Handle these specially as they are strings searched
212 # for during testing.
213 r"\| \bcount\b",
214 r"\| \bnot\b"]:
215 tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
216 if not tool_path:
217 # Warn, but still provide a substitution.
218 lit_config.note('Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
219 tool_path = config.llvm_tools_dir + '/' + tool_name
220 config.substitutions.append((pattern, tool_pipe + tool_path))
137 JUNKCHARS = r".-^/<"
138
139 required_tools = [
140 'lli', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov',
141 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', 'llvm-dsymutil',
142 'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-lib',
143 'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mcmarkup',
144 'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
145 'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj',
146 'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-tblgen',
147 'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
148 'FileCheck', 'yaml-bench', 'verify-uselistorder',
149 ToolFilter('bugpoint', post='-'),
150 ToolFilter('llc', pre=JUNKCHARS),
151 ToolFilter('llvm-symbolizer', pre=JUNKCHARS),
152 ToolFilter('opt', JUNKCHARS),
153 ToolFilter('sancov', pre=JUNKCHARS),
154 ToolFilter('sanstats', pre=JUNKCHARS),
155 # Handle these specially as they are strings searched for during testing.
156 ToolFilter(r'\| \bcount\b', verbatim=True),
157 ToolFilter(r'\| \bnot\b', verbatim=True)]
158
159 llvm_config.add_tool_substitutions(required_tools, config.llvm_tools_dir)
221160
222161 # For tools that are optional depending on the config, we won't warn
223162 # if they're missing.
224 for pattern in [r"\bllvm-go\b",
225 r"\bllvm-mt\b",
226 r"\bKaleidoscope-Ch3\b",
227 r"\bKaleidoscope-Ch4\b",
228 r"\bKaleidoscope-Ch5\b",
229 r"\bKaleidoscope-Ch6\b",
230 r"\bKaleidoscope-Ch7\b",
231 r"\bKaleidoscope-Ch8\b"]:
232 tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
233 if not tool_path:
234 # Provide a substitution anyway, for the sake of consistent errors.
235 tool_path = config.llvm_tools_dir + '/' + tool_name
236 config.substitutions.append((pattern, tool_pipe + tool_path))
237
163
164 optional_tools = [
165 'llvm-go', 'llvm-mt', 'Kaleidoscope-Ch3', 'Kaleidoscope-Ch4',
166 'Kaleidoscope-Ch5', 'Kaleidoscope-Ch6', 'Kaleidoscope-Ch7',
167 'Kaleidoscope-Ch8']
168 llvm_config.add_tool_substitutions(optional_tools, config.llvm_tools_dir,
169 warn_missing=False)
238170
239171 ### Targets
240172
None
1 from lit.llvm import config
1 import lit.util
2 import re
23
34 llvm_config = None
45
6 class ToolFilter(object):
7 """
8 String-like class used to build regex substitution patterns for
9 llvm tools. Handles things like adding word-boundary patterns,
10 and filtering characters from the beginning an end of a tool name
11 """
12
13 def __init__(self, name, pre=None, post=None, verbatim=False):
14 """
15 Construct a ToolFilter.
16
17 name: the literal name of the substitution to look for.
18
19 pre: If specified, the substitution will not find matches where
20 the character immediately preceding the word-boundary that begins
21 `name` is any of the characters in the string `pre`.
22
23 post: If specified, the substitution will not find matches where
24 the character immediately after the word-boundary that ends `name`
25 is any of the characters specified in the string `post`.
26
27 verbatim: If True, `name` is an exact regex that is passed to the
28 underlying substitution
29 """
30 if verbatim:
31 self.regex = name
32 return
33
34 def not_in(chars, where=''):
35 if not chars:
36 return ''
37 pattern_str = '|'.join(re.escape(x) for x in chars)
38 return r'(?{}!({}))'.format(where, pattern_str)
39
40 self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post)
41
42 def __str__(self):
43 return self.regex
44
45
546 def initialize(lit_config, test_config):
647 global llvm_config
48
749 llvm_config = config.LLVMConfig(lit_config, test_config)
850
135135 if name in self.config.environment:
136136 del self.config.environment[name]
137137
138 def feature_config(self, features, encoding = 'ascii'):
138 def get_process_output(self, command):
139 try:
140 cmd = subprocess.Popen(
141 command, stdout=subprocess.PIPE,
142 stderr=subprocess.PIPE, env=self.config.environment)
143 stdout, stderr = cmd.communicate()
144 stdout = lit.util.to_string(stdout)
145 stderr = lit.util.to_string(stderr)
146 return (stdout, stderr)
147 except OSError:
148 self.lit_config.fatal("Could not run process %s" % command)
149
150 def feature_config(self, features):
139151 # Ask llvm-config about the specified feature.
140152 arguments = [x for (x, _) in features]
141 try:
142 config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
143
144 llvm_config_cmd = subprocess.Popen(
145 [config_path] + arguments,
146 stdout = subprocess.PIPE,
147 env=self.config.environment)
148 except OSError:
149 self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
150
151 output, _ = llvm_config_cmd.communicate()
152 output = output.decode(encoding)
153 config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
154
155 output, _ = self.get_process_output([config_path] + arguments)
153156 lines = output.split('\n')
157
154158 for (feature_line, (_, patterns)) in zip(lines, features):
155159 # We should have either a callable or a dictionary. If it's a
156160 # dictionary, grep each key against the output and use the value if
162166 for (re_pattern, feature) in patterns.items():
163167 if re.search(re_pattern, feature_line):
164168 self.config.available_features.add(feature)
169
170
171 # Note that when substituting %clang_cc1 also fill in the include directory of
172 # the builtin headers. Those are part of even a freestanding environment, but
173 # Clang relies on the driver to locate them.
174 def get_clang_builtin_include_dir(self, clang):
175 # FIXME: Rather than just getting the version, we should have clang print
176 # out its resource dir here in an easy to scrape form.
177 clang_dir, _ = self.get_process_output([clang, '-print-file-name=include'])
178
179 if not clang_dir:
180 self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
181
182 clang_dir = clang_dir.strip()
183 if sys.platform in ['win32'] and not self.use_lit_shell:
184 # Don't pass dosish path separator to msys bash.exe.
185 clang_dir = clang_dir.replace('\\', '/')
186 # Ensure the result is an ascii string, across Python2.5+ - Python3.
187 return clang_dir
188
189 def make_itanium_abi_triple(self, triple):
190 m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
191 if not m:
192 self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
193 if m.group(3).lower() != 'win32':
194 # All non-win32 triples use the Itanium ABI.
195 return triple
196 return m.group(1) + '-' + m.group(2) + '-mingw32'
197
198 def make_msabi_triple(self, triple):
199 m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
200 if not m:
201 self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
202 isa = m.group(1).lower()
203 vendor = m.group(2).lower()
204 os = m.group(3).lower()
205 if os == 'win32':
206 # If the OS is win32, we're done.
207 return triple
208 if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
209 # For x86 ISAs, adjust the OS.
210 return isa + '-' + vendor + '-win32'
211 # -win32 is not supported for non-x86 targets; use a default.
212 return 'i686-pc-win32'
213
214 def add_tool_substitutions(self, tools, search_dirs, warn_missing = True):
215 if lit.util.is_string(search_dirs):
216 search_dirs = [search_dirs]
217
218 search_dirs = os.pathsep.join(search_dirs)
219 for tool in tools:
220 # Extract the tool name from the pattern. This relies on the tool
221 # name being surrounded by \b word match operators. If the
222 # pattern starts with "| ", include it in the string to be
223 # substituted.
224 if lit.util.is_string(tool):
225 tool = lit.util.make_word_regex(tool)
226 else:
227 tool = str(tool)
228
229 tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
230 tool)
231 if not tool_match:
232 continue
233
234 tool_pipe = tool_match.group(2)
235 tool_name = tool_match.group(4)
236 tool_path = lit.util.which(tool_name, search_dirs)
237 if not tool_path:
238 if warn_missing:
239 # Warn, but still provide a substitution.
240 self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs)
241 tool_path = self.config.llvm_tools_dir + '/' + tool_name
242
243 if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1':
244 tool_path += ' -verify-machineinstrs'
245 if tool_name == 'llvm-go':
246 exe = getattr(self.config, 'go_executable', None)
247 if exe:
248 tool_path += " go=" + exe
249
250 self.config.substitutions.append((tool, tool_pipe + tool_path))
3535 return False
3636 raise ValueError('"{}" is not a valid boolean'.format(value))
3737
38 def make_word_regex(word):
39 return r'\b' + word + r'\b'
3840
3941 def to_bytes(s):
4042 """Return the parameter as type 'bytes', possibly encoding it.