diff options
Diffstat (limited to 'utils/lit')
| -rw-r--r-- | utils/lit/lit/formats/__init__.py | 7 | ||||
| -rw-r--r-- | utils/lit/lit/formats/base.py | 157 | ||||
| -rw-r--r-- | utils/lit/lit/formats/googletest.py | 74 | ||||
| -rw-r--r-- | utils/lit/lit/formats/shtest.py | 40 | ||||
| -rw-r--r-- | utils/lit/lit/run.py | 218 | ||||
| -rw-r--r-- | utils/lit/lit/util.py | 126 |
6 files changed, 244 insertions, 378 deletions
diff --git a/utils/lit/lit/formats/__init__.py b/utils/lit/lit/formats/__init__.py index 3ff46e93ead2..7d14ca4b535a 100644 --- a/utils/lit/lit/formats/__init__.py +++ b/utils/lit/lit/formats/__init__.py @@ -1,8 +1,3 @@ -from lit.formats.base import ( # noqa: F401 - TestFormat, - FileBasedTest, - OneCommandPerFileTest -) - +from lit.formats.base import TestFormat # noqa: F401 from lit.formats.googletest import GoogleTest # noqa: F401 from lit.formats.shtest import ShTest # noqa: F401 diff --git a/utils/lit/lit/formats/base.py b/utils/lit/lit/formats/base.py index 6721d17e334e..baa9ff1d3b7d 100644 --- a/utils/lit/lit/formats/base.py +++ b/utils/lit/lit/formats/base.py @@ -1,117 +1,50 @@ -from __future__ import absolute_import -import os - -import lit.Test -import lit.util +import abc class TestFormat(object): - pass - -### - -class FileBasedTest(TestFormat): - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - source_path = testSuite.getSourcePath(path_in_suite) - for filename in os.listdir(source_path): - # Ignore dot files and excluded tests. - if (filename.startswith('.') or - filename in localConfig.excludes): - continue - - filepath = os.path.join(source_path, filename) - if not os.path.isdir(filepath): - base,ext = os.path.splitext(filename) - if ext in localConfig.suffixes: - yield lit.Test.Test(testSuite, path_in_suite + (filename,), - localConfig) - -### - -import re -import tempfile - -class OneCommandPerFileTest(TestFormat): - # FIXME: Refactor into generic test for running some command on a directory - # of inputs. - - def __init__(self, command, dir, recursive=False, - pattern=".*", useTempInput=False): - if isinstance(command, str): - self.command = [command] - else: - self.command = list(command) - if dir is not None: - dir = str(dir) - self.dir = dir - self.recursive = bool(recursive) - self.pattern = re.compile(pattern) - self.useTempInput = useTempInput - - def getTestsInDirectory(self, testSuite, path_in_suite, - litConfig, localConfig): - dir = self.dir - if dir is None: - dir = testSuite.getSourcePath(path_in_suite) - - for dirname,subdirs,filenames in os.walk(dir): - if not self.recursive: - subdirs[:] = [] - - subdirs[:] = [d for d in subdirs - if (d != '.svn' and - d not in localConfig.excludes)] - - for filename in filenames: - if (filename.startswith('.') or - not self.pattern.match(filename) or - filename in localConfig.excludes): - continue - - path = os.path.join(dirname,filename) - suffix = path[len(dir):] - if suffix.startswith(os.sep): - suffix = suffix[1:] - test = lit.Test.Test( - testSuite, path_in_suite + tuple(suffix.split(os.sep)), - localConfig) - # FIXME: Hack? - test.source_path = path - yield test - - def createTempInput(self, tmp, test): - raise NotImplementedError('This is an abstract method.') - + """Base class for test formats. + + A TestFormat encapsulates logic for finding and executing a certain type of + test. For example, a subclass FooTestFormat would contain the logic for + finding tests written in the 'Foo' format, and the logic for running a + single one. + + TestFormat is an Abstract Base Class (ABC). It uses the Python abc.ABCMeta + type and associated @abc.abstractmethod decorator. Together, these provide + subclass behaviour which is notionally similar to C++ pure virtual classes: + only subclasses which implement all abstract methods can be instantiated + (the implementation may come from an intermediate base). + + For details on ABCs, see: https://docs.python.org/2/library/abc.html. Note + that Python ABCs have extensive abilities beyond what is used here. For + TestFormat, we only care about enforcing that abstract methods are + implemented. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, + localConfig): + """Finds tests of this format in the given directory. + + Args: + testSuite: a Test.TestSuite object. + path_in_suite: the subpath under testSuite to look for tests. + litConfig: the LitConfig for the test suite. + localConfig: a LitConfig with local specializations. + + Returns: + An iterable of Test.Test objects. + """ + + @abc.abstractmethod def execute(self, test, litConfig): - if test.config.unsupported: - return (lit.Test.UNSUPPORTED, 'Test is unsupported') - - cmd = list(self.command) - - # If using temp input, create a temporary file and hand it to the - # subclass. - if self.useTempInput: - tmp = tempfile.NamedTemporaryFile(suffix='.cpp') - self.createTempInput(tmp, test) - tmp.flush() - cmd.append(tmp.name) - elif hasattr(test, 'source_path'): - cmd.append(test.source_path) - else: - cmd.append(test.getSourcePath()) - - out, err, exitCode = lit.util.executeCommand(cmd) - - diags = out + err - if not exitCode and not diags.strip(): - return lit.Test.PASS,'' + """Runs the given 'test', which is of this format. - # Try to include some useful information. - report = """Command: %s\n""" % ' '.join(["'%s'" % a - for a in cmd]) - if self.useTempInput: - report += """Temporary File: %s\n""" % tmp.name - report += "--\n%s--\n""" % open(tmp.name).read() - report += """Output:\n--\n%s--""" % diags + Args: + test: a Test.Test object describing the test to run. + litConfig: the LitConfig for the test suite. - return lit.Test.FAIL, report + Returns: + A tuple of (status:Test.ResultCode, message:str) + """ diff --git a/utils/lit/lit/formats/googletest.py b/utils/lit/lit/formats/googletest.py index 29a92c4e960b..b683f7c7db8e 100644 --- a/utils/lit/lit/formats/googletest.py +++ b/utils/lit/lit/formats/googletest.py @@ -11,8 +11,8 @@ from .base import TestFormat kIsWindows = sys.platform in ['win32', 'cygwin'] class GoogleTest(TestFormat): - def __init__(self, test_sub_dir, test_suffix): - self.test_sub_dir = os.path.normcase(str(test_sub_dir)).split(';') + def __init__(self, test_sub_dirs, test_suffix): + self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(';') self.test_suffix = str(test_suffix) # On Windows, assume tests will also end in '.exe'. @@ -30,19 +30,24 @@ class GoogleTest(TestFormat): localConfig: TestingConfig instance""" try: - lines = lit.util.capture([path, '--gtest_list_tests'], - env=localConfig.environment) - if kIsWindows: - lines = lines.replace('\r', '') - lines = lines.split('\n') - except Exception as exc: - out = exc.output if isinstance(exc, subprocess.CalledProcessError) else '' - litConfig.warning("unable to discover google-tests in %r: %s. Process output: %s" - % (path, sys.exc_info()[1], out)) + output = subprocess.check_output([path, '--gtest_list_tests'], + env=localConfig.environment) + except subprocess.CalledProcessError as exc: + litConfig.warning( + "unable to discover google-tests in %r: %s. Process output: %s" + % (path, sys.exc_info()[1], exc.output)) raise StopIteration nested_tests = [] - for ln in lines: + for ln in output.splitlines(False): # Don't keep newlines. + ln = lit.util.to_string(ln) + + if 'Running main() from gtest_main.cc' in ln: + # Upstream googletest prints this to stdout prior to running + # tests. LLVM removed that print statement in r61540, but we + # handle it here in case upstream googletest is being used. + continue + # The test name list includes trailing comments beginning with # a '#' on some lines, so skip those. We don't support test names # that use escaping to embed '#' into their name as the names come @@ -52,12 +57,6 @@ class GoogleTest(TestFormat): if not ln.lstrip(): continue - if 'Running main() from gtest_main.cc' in ln: - # Upstream googletest prints this to stdout prior to running - # tests. LLVM removed that print statement in r61540, but we - # handle it here in case upstream googletest is being used. - continue - index = 0 while ln[index*2:index*2+2] == ' ': index += 1 @@ -75,38 +74,19 @@ class GoogleTest(TestFormat): else: yield ''.join(nested_tests) + ln - # Note: path_in_suite should not include the executable name. - def getTestsInExecutable(self, testSuite, path_in_suite, execpath, - litConfig, localConfig): - if not execpath.endswith(self.test_suffix): - return - (dirname, basename) = os.path.split(execpath) - # Discover the tests in this executable. - for testname in self.getGTestTests(execpath, litConfig, localConfig): - testPath = path_in_suite + (basename, testname) - yield lit.Test.Test(testSuite, testPath, localConfig, file_path=execpath) - def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig): source_path = testSuite.getSourcePath(path_in_suite) - for filename in os.listdir(source_path): - filepath = os.path.join(source_path, filename) - if os.path.isdir(filepath): - # Iterate over executables in a directory. - if not os.path.normcase(filename) in self.test_sub_dir: - continue - dirpath_in_suite = path_in_suite + (filename, ) - for subfilename in os.listdir(filepath): - execpath = os.path.join(filepath, subfilename) - for test in self.getTestsInExecutable( - testSuite, dirpath_in_suite, execpath, - litConfig, localConfig): - yield test - elif ('.' in self.test_sub_dir): - for test in self.getTestsInExecutable( - testSuite, path_in_suite, filepath, - litConfig, localConfig): - yield test + for subdir in self.test_sub_dirs: + for fn in lit.util.listdir_files(os.path.join(source_path, subdir), + suffixes={self.test_suffix}): + # Discover the tests in this executable. + execpath = os.path.join(source_path, subdir, fn) + testnames = self.getGTestTests(execpath, litConfig, localConfig) + for testname in testnames: + testPath = path_in_suite + (subdir, fn, testname) + yield lit.Test.Test(testSuite, testPath, localConfig, + file_path=execpath) def execute(self, test, litConfig): testPath,testName = os.path.split(test.getSourcePath()) diff --git a/utils/lit/lit/formats/shtest.py b/utils/lit/lit/formats/shtest.py index 30a6a3310b01..01ecd192092e 100644 --- a/utils/lit/lit/formats/shtest.py +++ b/utils/lit/lit/formats/shtest.py @@ -1,12 +1,48 @@ from __future__ import absolute_import +import os + +import lit.Test import lit.TestRunner -from .base import FileBasedTest +import lit.util +from .base import TestFormat + +class ShTest(TestFormat): + """ShTest is a format with one file per test. + + This is the primary format for regression tests as described in the LLVM + testing guide: + + http://llvm.org/docs/TestingGuide.html + + The ShTest files contain some number of shell-like command pipelines, along + with assertions about what should be in the output. + """ -class ShTest(FileBasedTest): def __init__(self, execute_external = False): + """Initializer. + + The 'execute_external' argument controls whether lit uses its internal + logic for command pipelines, or passes the command to a shell + subprocess. + + Args: + execute_external: (optional) If true, use shell subprocesses instead + of lit's internal pipeline logic. + """ self.execute_external = execute_external + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + """Yields test files matching 'suffixes' from the localConfig.""" + file_matches = lit.util.listdir_files( + testSuite.getSourcePath(path_in_suite), + localConfig.suffixes, localConfig.excludes) + for filename in file_matches: + yield lit.Test.Test(testSuite, path_in_suite + (filename,), + localConfig) + def execute(self, test, litConfig): + """Interprets and runs the given test file, and returns the result.""" return lit.TestRunner.executeShTest(test, litConfig, self.execute_external) diff --git a/utils/lit/lit/run.py b/utils/lit/lit/run.py index aa4fdc18b877..1290c142c834 100644 --- a/utils/lit/lit/run.py +++ b/utils/lit/lit/run.py @@ -24,140 +24,6 @@ def abort_now(): else: os.kill(0, 9) -### -# Test Execution Implementation - -class LockedValue(object): - def __init__(self, value): - self.lock = threading.Lock() - self._value = value - - def _get_value(self): - self.lock.acquire() - try: - return self._value - finally: - self.lock.release() - - def _set_value(self, value): - self.lock.acquire() - try: - self._value = value - finally: - self.lock.release() - - value = property(_get_value, _set_value) - -class TestProvider(object): - def __init__(self, queue_impl, canceled_flag): - self.canceled_flag = canceled_flag - - # Create a shared queue to provide the test indices. - self.queue = queue_impl() - - def queue_tests(self, tests, num_jobs): - for i in range(len(tests)): - self.queue.put(i) - for i in range(num_jobs): - self.queue.put(None) - - def cancel(self): - self.canceled_flag.value = 1 - - def get(self): - # Check if we are canceled. - if self.canceled_flag.value: - return None - - # Otherwise take the next test. - return self.queue.get() - -class Tester(object): - def __init__(self, run_instance, provider, consumer): - self.run_instance = run_instance - self.provider = provider - self.consumer = consumer - - def run(self): - while True: - item = self.provider.get() - if item is None: - break - self.run_test(item) - self.consumer.task_finished() - - def run_test(self, test_index): - test = self.run_instance.tests[test_index] - try: - execute_test(test, self.run_instance.lit_config, - self.run_instance.parallelism_semaphores) - except KeyboardInterrupt: - # This is a sad hack. Unfortunately subprocess goes - # bonkers with ctrl-c and we start forking merrily. - print('\nCtrl-C detected, goodbye.') - abort_now() - self.consumer.update(test_index, test) - -class ThreadResultsConsumer(object): - def __init__(self, display): - self.display = display - self.lock = threading.Lock() - - def update(self, test_index, test): - self.lock.acquire() - try: - self.display.update(test) - finally: - self.lock.release() - - def task_finished(self): - pass - - def handle_results(self): - pass - -class MultiprocessResultsConsumer(object): - def __init__(self, run, display, num_jobs): - self.run = run - self.display = display - self.num_jobs = num_jobs - self.queue = multiprocessing.Queue() - - def update(self, test_index, test): - # This method is called in the child processes, and communicates the - # results to the actual display implementation via an output queue. - self.queue.put((test_index, test.result)) - - def task_finished(self): - # This method is called in the child processes, and communicates that - # individual tasks are complete. - self.queue.put(None) - - def handle_results(self): - # This method is called in the parent, and consumes the results from the - # output queue and dispatches to the actual display. The method will - # complete after each of num_jobs tasks has signalled completion. - completed = 0 - while completed != self.num_jobs: - # Wait for a result item. - item = self.queue.get() - if item is None: - completed += 1 - continue - - # Update the test result in the parent process. - index,result = item - test = self.run.tests[index] - test.result = result - - self.display.update(test) - -def run_one_tester(run, provider, display): - tester = Tester(run, provider, display) - tester.run() - -### - class _Display(object): def __init__(self, display, provider, maxFailures): self.display = display @@ -170,47 +36,6 @@ class _Display(object): if self.failedCount == self.maxFailures: self.provider.cancel() -def handleFailures(provider, consumer, maxFailures): - consumer.display = _Display(consumer.display, provider, maxFailures) - -def execute_test(test, lit_config, parallelism_semaphores): - """Execute one test""" - pg = test.config.parallelism_group - if callable(pg): - pg = pg(test) - - result = None - semaphore = None - try: - if pg: - semaphore = parallelism_semaphores[pg] - if semaphore: - semaphore.acquire() - start_time = time.time() - result = test.config.test_format.execute(test, lit_config) - # Support deprecated result from execute() which returned the result - # code and additional output as a tuple. - if isinstance(result, tuple): - code, output = result - result = lit.Test.Result(code, output) - elif not isinstance(result, lit.Test.Result): - raise ValueError("unexpected result from test execution") - result.elapsed = time.time() - start_time - except KeyboardInterrupt: - raise - except: - if lit_config.debug: - raise - output = 'Exception during script execution:\n' - output += traceback.format_exc() - output += '\n' - result = lit.Test.Result(lit.Test.UNRESOLVED, output) - finally: - if semaphore: - semaphore.release() - - test.setResult(result) - class Run(object): """ This class represents a concrete, configured testing run. @@ -221,7 +46,8 @@ class Run(object): self.tests = tests def execute_test(self, test): - return execute_test(test, self.lit_config, self.parallelism_semaphores) + return _execute_test_impl(test, self.lit_config, + self.parallelism_semaphores) def execute_tests(self, display, jobs, max_time=None): """ @@ -350,6 +176,44 @@ class Run(object): self.failure_count == self.lit_config.maxFailures: self.hit_max_failures = True +def _execute_test_impl(test, lit_config, parallelism_semaphores): + """Execute one test""" + pg = test.config.parallelism_group + if callable(pg): + pg = pg(test) + + result = None + semaphore = None + try: + if pg: + semaphore = parallelism_semaphores[pg] + if semaphore: + semaphore.acquire() + start_time = time.time() + result = test.config.test_format.execute(test, lit_config) + # Support deprecated result from execute() which returned the result + # code and additional output as a tuple. + if isinstance(result, tuple): + code, output = result + result = lit.Test.Result(code, output) + elif not isinstance(result, lit.Test.Result): + raise ValueError("unexpected result from test execution") + result.elapsed = time.time() - start_time + except KeyboardInterrupt: + raise + except: + if lit_config.debug: + raise + output = 'Exception during script execution:\n' + output += traceback.format_exc() + output += '\n' + result = lit.Test.Result(lit.Test.UNRESOLVED, output) + finally: + if semaphore: + semaphore.release() + + test.setResult(result) + child_lit_config = None child_parallelism_semaphores = None @@ -375,7 +239,7 @@ def worker_run_one_test(test_index, test): the display. """ try: - execute_test(test, child_lit_config, child_parallelism_semaphores) + _execute_test_impl(test, child_lit_config, child_parallelism_semaphores) return (test_index, test) except KeyboardInterrupt as e: # If a worker process gets an interrupt, abort it immediately. diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py index 8991588a868d..1819d4d1c34f 100644 --- a/utils/lit/lit/util.py +++ b/utils/lit/lit/util.py @@ -8,24 +8,57 @@ import subprocess import sys import threading -def to_bytes(str): - # Encode to UTF-8 to get binary data. - if isinstance(str, bytes): - return str - return str.encode('utf-8') - -def to_string(bytes): - if isinstance(bytes, str): - return bytes - return to_bytes(bytes) - -def convert_string(bytes): +def to_bytes(s): + """Return the parameter as type 'bytes', possibly encoding it. + + In Python2, the 'bytes' type is the same as 'str'. In Python3, they are + distinct. + """ + if isinstance(s, bytes): + # In Python2, this branch is taken for both 'str' and 'bytes'. + # In Python3, this branch is taken only for 'bytes'. + return s + # In Python2, 's' is a 'unicode' object. + # In Python3, 's' is a 'str' object. + # Encode to UTF-8 to get 'bytes' data. + return s.encode('utf-8') + +def to_string(b): + """Return the parameter as type 'str', possibly encoding it. + + In Python2, the 'str' type is the same as 'bytes'. In Python3, the + 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is + distinct. + """ + if isinstance(b, str): + # In Python2, this branch is taken for types 'str' and 'bytes'. + # In Python3, this branch is taken only for 'str'. + return b + if isinstance(b, bytes): + # In Python2, this branch is never taken ('bytes' is handled as 'str'). + # In Python3, this is true only for 'bytes'. + try: + return b.decode('utf-8') + except UnicodeDecodeError: + # If the value is not valid Unicode, return the default + # repr-line encoding. + return str(b) + + # By this point, here's what we *don't* have: + # + # - In Python2: + # - 'str' or 'bytes' (1st branch above) + # - In Python3: + # - 'str' (1st branch above) + # - 'bytes' (2nd branch above) + # + # The last type we might expect is the Python2 'unicode' type. There is no + # 'unicode' type in Python3 (all the Python3 cases were already handled). In + # order to get a 'str' object, we need to encode the 'unicode' object. try: - return to_string(bytes.decode('utf-8')) - except AttributeError: # 'str' object has no attribute 'decode'. - return str(bytes) - except UnicodeError: - return str(bytes) + return b.encode('utf-8') + except AttributeError: + raise TypeError('not sure how to convert %s to %s' % (type(b), str)) def detectCPUs(): """ @@ -39,7 +72,8 @@ def detectCPUs(): if isinstance(ncpus, int) and ncpus > 0: return ncpus else: # OSX: - return int(capture(['sysctl', '-n', 'hw.ncpu'])) + return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'], + stderr=subprocess.STDOUT)) # Windows: if "NUMBER_OF_PROCESSORS" in os.environ: ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) @@ -67,20 +101,44 @@ def mkdir_p(path): if e.errno != errno.EEXIST: raise -def capture(args, env=None): - """capture(command) - Run the given command (or argv list) in a shell and - return the standard output. Raises a CalledProcessError if the command - exits with a non-zero status.""" - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env) - out, err = p.communicate() - out = convert_string(out) - err = convert_string(err) - if p.returncode != 0: - raise subprocess.CalledProcessError(cmd=args, - returncode=p.returncode, - output="{}\n{}".format(out, err)) - return out +def listdir_files(dirname, suffixes=None, exclude_filenames=None): + """Yields files in a directory. + + Filenames that are not excluded by rules below are yielded one at a time, as + basenames (i.e., without dirname). + + Files starting with '.' are always skipped. + + If 'suffixes' is not None, then only filenames ending with one of its + members will be yielded. These can be extensions, like '.exe', or strings, + like 'Test'. (It is a lexicographic check; so an empty sequence will yield + nothing, but a single empty string will yield all filenames.) + + If 'exclude_filenames' is not None, then none of the file basenames in it + will be yielded. + + If specified, the containers for 'suffixes' and 'exclude_filenames' must + support membership checking for strs. + + Args: + dirname: a directory path. + suffixes: (optional) a sequence of strings (set, list, etc.). + exclude_filenames: (optional) a sequence of strings. + + Yields: + Filenames as returned by os.listdir (generally, str). + """ + if exclude_filenames is None: + exclude_filenames = set() + if suffixes is None: + suffixes = {''} + for filename in os.listdir(dirname): + if (os.path.isdir(os.path.join(dirname, filename)) or + filename.startswith('.') or + filename in exclude_filenames or + not any(filename.endswith(sfx) for sfx in suffixes)): + continue + yield filename def which(command, paths = None): """which(command, [paths]) - Look up the given command in the paths string @@ -233,8 +291,8 @@ def executeCommand(command, cwd=None, env=None, input=None, timeout=0): timerObject.cancel() # Ensure the resulting output is always of string type. - out = convert_string(out) - err = convert_string(err) + out = to_string(out) + err = to_string(err) if hitTimeOut[0]: raise ExecuteCommandTimeoutException( |