llvm.org GIT mirror llvm / 5a321ee
[lit] Clean output directories before running tests. Presently lit leaks files in the tests' output directories. Specifically, if a test creates output files, lit makes no effort to remove them prior to the next test run. This is problematic because it leads to false positives whenever a test passes because stale files were present. In general it is a source of flakiness that should be removed. This patch addresses this by building the list of all test directories that are part of the current run set, and then deleting those directories and recreating them anew. This gives each test a clean baseline to start from. Differential Revision: https://reviews.llvm.org/D34732 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@306832 91177308-0d34-0410-b5e6-96231b3b80d8 Zachary Turner 2 years ago
15 changed file(s) with 104 addition(s) and 74 deletion(s). Raw diff Collapse all Expand all
0 void f1();
1 __attribute__((always_inline)) void f2() {
2 f1();
3 }
4 void f3() {
5 f2();
6 }
0 void f1();
1 inline __attribute__((always_inline)) void f2() {
2 f1();
3 }
4 void f3() {
5 f2();
6 }
1919 RUN: echo "%p/Inputs/fission-ranges.elf-x86_64 0x720" >> %t.input
2020 RUN: echo "%p/Inputs/arange-overlap.elf-x86_64 0x714" >> %t.input
2121 RUN: cp %p/Inputs/split-dwarf-test.dwo %T
22 RUN: echo "%p/Inputs/split-dwarf-test 0x4005d4" >> %t.input
23 RUN: echo "%p/Inputs/split-dwarf-test 0x4005c4" >> %t.input
22 RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
23 RUN: echo "%p/Inputs/split-dwarf-test 0x4004f0" >> %t.input
2424 RUN: echo "%p/Inputs/cross-cu-inlining.x86_64-macho.o 0x17" >> %t.input
2525 RUN: cp %p/Inputs/split-dwarf-multiple-cu.dwo %T
2626 RUN: echo "%p/Inputs/split-dwarf-multiple-cu.o 0x4" >> %t.input
140140 CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:11:0
141141
142142 CHECK: f2
143 CHECK-NEXT: b.cpp:3:3
143 CHECK-NEXT: split-dwarf-multiple-cu2.cpp:3:3
144144 CHECK-NEXT: f3
145 CHECK-NEXT: b.cpp:6:0
145 CHECK-NEXT: split-dwarf-multiple-cu2.cpp:6:0
146146
147147 CHECK: f2
148148 CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:3:3
None RUN: echo TA > %T/TA.txt
1 RUN: echo TB > %T/TB.txt
2 RUN: echo TAB > %T/TAB.txt
3
4 RUN: echo %T/TA* | FileCheck -check-prefix=STAR %s
5 RUN: echo %T/'TA'* | FileCheck -check-prefix=STAR %s
6 RUN: echo %T/T'A'* | FileCheck -check-prefix=STAR %s
7
8 RUN: echo %T/T?.txt | FileCheck -check-prefix=QUESTION %s
9 RUN: echo %T/'T'?.txt | FileCheck -check-prefix=QUESTION %s
10
11 RUN: echo %T/T??.txt | FileCheck -check-prefix=QUESTION2 %s
12 RUN: echo %T/'T'??.txt | FileCheck -check-prefix=QUESTION2 %s
13
14 RUN: echo 'T*' 'T?.txt' 'T??.txt' | FileCheck -check-prefix=QUOTEDARGS %s
15
16 STAR-NOT: TB.txt
17 STAR: {{(TA.txt.*TAB.txt|TAB.txt.*TA.txt)}}
18
19 QUESTION-NOT: TAB.txt
20 QUESTION: {{(TA.txt.*TB.txt|TB.txt.*TA.txt)}}
21
22 QUESTION2-NOT: TA.txt
23 QUESTION2-NOT: TB.txt
24 QUESTION2: TAB.txt
25
26 QUOTEDARGS-NOT: .txt
27 QUOTEDARGS: T* T?.txt T??.txt
0 RUN: echo XXA > %T/XXA.txt
1 RUN: echo XXB > %T/XXB.txt
2 RUN: echo XXAB > %T/XXAB.txt
3
4 RUN: echo %T/XXA* | FileCheck -check-prefix=STAR %s
5 RUN: echo %T/'XXA'* | FileCheck -check-prefix=STAR %s
6 RUN: echo %T/XX'A'* | FileCheck -check-prefix=STAR %s
7
8 RUN: echo %T/XX?.txt | FileCheck -check-prefix=QUESTION %s
9 RUN: echo %T/'XX'?.txt | FileCheck -check-prefix=QUESTION %s
10
11 RUN: echo %T/XX??.txt | FileCheck -check-prefix=QUESTION2 %s
12 RUN: echo %T/'XX'??.txt | FileCheck -check-prefix=QUESTION2 %s
13
14 RUN: echo 'XX*' 'XX?.txt' 'XX??.txt' | FileCheck -check-prefix=QUOTEDARGS %s
15
16 STAR-NOT: XXB.txt
17 STAR: {{(XXA.txt.*XXAB.txt|XXAB.txt.*XXA.txt)}}
18
19 QUESTION-NOT: XXAB.txt
20 QUESTION: {{(XXA.txt.*XXB.txt|XXB.txt.*XXA.txt)}}
21
22 QUESTION2-NOT: XXA.txt
23 QUESTION2-NOT: XXB.txt
24 QUESTION2: XXAB.txt
25
26 QUOTEDARGS-NOT: .txt
27 QUOTEDARGS: XX* XX?.txt XX??.txt
3838 config.environment[symbolizer] = os.environ[symbolizer]
3939
4040 # Win32 seeks DLLs along %PATH%.
41 if sys.platform in ['win32', 'cygwin'] and os.path.isdir(config.shlibdir):
42 config.environment['PATH'] = os.path.pathsep.join((
43 config.shlibdir, config.environment['PATH']))
41 if sys.platform in ['win32', 'cygwin']:
42 shlibdir = getattr(config, 'shlibdir', None)
43 if shlibdir is not None and os.path.isdir(shlibdir):
44 config.environment['PATH'] = os.path.pathsep.join((
45 config.shlibdir, config.environment['PATH']))
4446
4547 # Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
4648 if sys.platform == 'win32' and 'SYSTEMDRIVE' in os.environ:
171171 return os.path.join(self.source_root, *components)
172172
173173 def getExecPath(self, components):
174 return os.path.join(self.exec_root, *components)
174 return os.path.join(self.exec_root, "Output", *components)
175175
176176 class Test:
177177 """Test - Information on a single test instance."""
221221 # Syntax error in an XFAIL line.
222222 self.result.code = UNRESOLVED
223223 self.result.output = str(e)
224
224
225225 def getFullName(self):
226226 return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
227
228 def getTestBaseName(self):
229 return self.path_in_suite[-1]
227230
228231 def getFilePath(self):
229232 if self.file_path:
233236 def getSourcePath(self):
234237 return self.suite.getSourcePath(self.path_in_suite)
235238
236 def getExecPath(self):
237 return self.suite.getExecPath(self.path_in_suite)
239 def getTempFilePrefix(self):
240 return self.suite.getExecPath(self.path_in_suite) + ".tmp"
241
242 def getTempFileDir(self):
243 return os.path.dirname(self.getTempFilePrefix())
238244
239245 def isExpectedToFail(self):
240246 """
346352 safe_name = self.suite.name.replace(".","-")
347353
348354 if safe_test_path:
349 class_name = safe_name + "." + "/".join(safe_test_path)
355 class_name = safe_name + "." + "/".join(safe_test_path)
350356 else:
351357 class_name = safe_name + "." + safe_name
352358
122122 self.exitCode = exitCode
123123 self.timeoutReached = timeoutReached
124124 self.outputFiles = list(outputFiles)
125
125
126126 def executeShCmd(cmd, shenv, results, timeout=0):
127127 """
128128 Wrapper around _executeShCmd that handles
500500 data = None
501501 if data is not None:
502502 output_files.append((name, path, data))
503
503
504504 results.append(ShellCommandResult(
505505 cmd.commands[i], out, err, res, timeoutHelper.timeoutReached(),
506506 output_files))
572572 else:
573573 out += data
574574 out += "\n"
575
575
576576 if result.stdout.strip():
577577 out += '# command output:\n%s\n' % (result.stdout,)
578578 if result.stderr.strip():
689689 finally:
690690 f.close()
691691
692 def getTempPaths(test):
693 """Get the temporary location, this is always relative to the test suite
694 root, not test source root."""
695 execpath = test.getExecPath()
696 execdir,execbase = os.path.split(execpath)
697 tmpDir = os.path.join(execdir, 'Output')
698 tmpBase = os.path.join(tmpDir, execbase)
699 return tmpDir, tmpBase
700
701 def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
692 def getDefaultSubstitutions(test, normalize_slashes=False):
702693 sourcepath = test.getSourcePath()
703694 sourcedir = os.path.dirname(sourcepath)
695 tmpDir = test.getTempFileDir()
696 tmpPrefix = test.getTempFilePrefix()
697 baseName = test.getTestBaseName()
704698
705699 # Normalize slashes, if requested.
706700 if normalize_slashes:
707701 sourcepath = sourcepath.replace('\\', '/')
708702 sourcedir = sourcedir.replace('\\', '/')
709703 tmpDir = tmpDir.replace('\\', '/')
710 tmpBase = tmpBase.replace('\\', '/')
711704
712705 # We use #_MARKER_# to hide %% while we do the other substitutions.
713706 substitutions = []
714707 substitutions.extend([('%%', '#_MARKER_#')])
715708 substitutions.extend(test.config.substitutions)
716 tmpName = tmpBase + '.tmp'
717 baseName = os.path.basename(tmpBase)
718709 substitutions.extend([('%s', sourcepath),
719710 ('%S', sourcedir),
720711 ('%p', sourcedir),
721712 ('%{pathsep}', os.pathsep),
722 ('%t', tmpName),
713 ('%t', tmpPrefix),
723714 ('%basename_t', baseName),
724715 ('%T', tmpDir),
725716 ('#_MARKER_#', '%')])
729720 ('%/s', sourcepath.replace('\\', '/')),
730721 ('%/S', sourcedir.replace('\\', '/')),
731722 ('%/p', sourcedir.replace('\\', '/')),
732 ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
723 ('%/t', tmpPrefix.replace('\\', '/')),
733724 ('%/T', tmpDir.replace('\\', '/')),
734725 ])
735726
739730 ('%:s', re.sub(r'^(.):', r'\1', sourcepath)),
740731 ('%:S', re.sub(r'^(.):', r'\1', sourcedir)),
741732 ('%:p', re.sub(r'^(.):', r'\1', sourcedir)),
742 ('%:t', re.sub(r'^(.):', r'\1', tmpBase) + '.tmp'),
733 ('%:t', re.sub(r'^(.):', r'\1', tmpPrefix)),
743734 ('%:T', re.sub(r'^(.):', r'\1', tmpDir)),
744735 ])
745736 else:
747738 ('%:s', sourcepath),
748739 ('%:S', sourcedir),
749740 ('%:p', sourcedir),
750 ('%:t', tmpBase + '.tmp'),
741 ('%:t', tmpPrefix),
751742 ('%:T', tmpDir),
752743 ])
753744 return substitutions
778769 TAG: A keyword taking no value. Ex 'END.'
779770 COMMAND: A keyword taking a list of shell commands. Ex 'RUN:'
780771 LIST: A keyword taking a comma-separated list of values.
781 BOOLEAN_EXPR: A keyword taking a comma-separated list of
772 BOOLEAN_EXPR: A keyword taking a comma-separated list of
782773 boolean expressions. Ex 'XFAIL:'
783774 CUSTOM: A keyword with custom parsing semantics.
784775 """
950941 IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
951942 initial_value=test.requires),
952943 IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.CUSTOM,
953 IntegratedTestKeywordParser._handleRequiresAny,
954 initial_value=test.requires),
944 IntegratedTestKeywordParser._handleRequiresAny,
945 initial_value=test.requires),
955946 IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
956947 initial_value=test.unsupported),
957948 IntegratedTestKeywordParser('END.', ParserKind.TAG)
958949 ]
959950 keyword_parsers = {p.keyword: p for p in builtin_parsers}
960
951
961952 # Install user-defined additional parsers.
962953 for parser in additional_parsers:
963954 if not isinstance(parser, IntegratedTestKeywordParser):
967958 raise ValueError("Parser for keyword '%s' already exists"
968959 % parser.keyword)
969960 keyword_parsers[parser.keyword] = parser
970
961
971962 # Collect the test lines from the script.
972963 sourcepath = test.getSourcePath()
973964 for line_number, command_type, ln in \
10131004
10141005 return script
10151006
1016
10171007 def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
1018 # Create the output directory if it does not already exist.
1019 lit.util.mkdir_p(os.path.dirname(tmpBase))
1020
1021 execdir = os.path.dirname(test.getExecPath())
1008 execdir = os.path.dirname(test.getTempFileDir())
10221009 if useExternalSh:
10231010 res = executeScript(test, litConfig, tmpBase, script, execdir)
10241011 else:
10621049 return script
10631050 if litConfig.noExecute:
10641051 return lit.Test.Result(Test.PASS)
1065
1066 tmpDir, tmpBase = getTempPaths(test)
10671052 substitutions = list(extra_substitutions)
1068 substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
1053 substitutions += getDefaultSubstitutions(test,
10691054 normalize_slashes=useExternalSh)
10701055 script = applySubstitutions(script, substitutions)
10711056
10741059 if hasattr(test.config, 'test_retry_attempts'):
10751060 attempts += test.config.test_retry_attempts
10761061 for i in range(attempts):
1077 res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
1062 res = _runShTest(test, litConfig, useExternalSh, script,
1063 test.getTempFilePrefix())
10781064 if res.code != Test.FAIL:
10791065 break
10801066 # If we had to run the test more than once, count it as a flaky pass. These
0 import os
1 import shutil
12 import sys
23 import threading
34 import time
7273 # Don't do anything if we aren't going to run any tests.
7374 if not self.tests or jobs == 0:
7475 return
76
77 # Create fresh output directories for each test we're going to run.
78 # This guarantees that test runs will not remnants of previous test
79 # runs' output.
80 clean_paths = set()
81 for test in self.tests:
82 clean_paths.add(os.path.normpath(test.getTempFileDir()))
83 clean_paths = list(clean_paths)
84 # Sort by number of path components, to ensure that parent directories
85 # get deleted and re-created before child directories.
86 clean_paths.sort(key=lambda x: len(x.split(os.sep)))
87 for base in clean_paths:
88 if os.path.exists(base):
89 if not os.path.islink(base) and os.path.isdir(base):
90 shutil.rmtree(base, True)
91 else:
92 os.unlink(os.path)
93 if not os.path.exists(base):
94 lit.util.mkdir_p(base)
7595
7696 # Set up semaphores to limit parallelism of certain classes of tests.
7797 # For example, some ASan tests require lots of virtual memory and run