from __future__ import absolute_import from __future__ import print_function # System modules from distutils.version import LooseVersion, StrictVersion from functools import wraps import os import re import sys import tempfile # Third-party modules import six import unittest2 # LLDB modules import use_lldb_suite import lldb from . import configuration from . import test_categories from lldbsuite.test_event.event_builder import EventBuilder from lldbsuite.support import funcutils from lldbsuite.test import lldbplatform from lldbsuite.test import lldbplatformutil class DecorateMode: Skip, Xfail = range(2) # You can use no_match to reverse the test of the conditional that is used to match keyword # arguments in the skip / xfail decorators. If oslist=["windows", "linux"] skips windows # and linux, oslist=no_match(["windows", "linux"]) skips *unless* windows or linux. class no_match: def __init__(self, item): self.item = item def _check_expected_version(comparison, expected, actual): def fn_leq(x,y): return x <= y def fn_less(x,y): return x < y def fn_geq(x,y): return x >= y def fn_greater(x,y): return x > y def fn_eq(x,y): return x == y def fn_neq(x,y): return x != y op_lookup = { "==": fn_eq, "=": fn_eq, "!=": fn_neq, "<>": fn_neq, ">": fn_greater, "<": fn_less, ">=": fn_geq, "<=": fn_leq } expected_str = '.'.join([str(x) for x in expected]) actual_str = '.'.join([str(x) for x in actual]) return op_lookup[comparison](LooseVersion(actual_str), LooseVersion(expected_str)) def _match_decorator_property(expected, actual): if actual is None or expected is None: return True if isinstance(expected, no_match): return not _match_decorator_property(expected.item, actual) elif isinstance(expected, (re._pattern_type,)+six.string_types): return re.search(expected, actual) is not None elif hasattr(expected, "__iter__"): return any([x is not None and _match_decorator_property(x, actual) for x in expected]) else: return expected == actual def expectedFailure(expected_fn, bugnumber=None): def expectedFailure_impl(func): if isinstance(func, type) and issubclass(func, unittest2.TestCase): raise Exception("Decorator can only be used to decorate a test method") @wraps(func) def wrapper(*args, **kwargs): self = args[0] if funcutils.requires_self(expected_fn): xfail_reason = expected_fn(self) else: xfail_reason = expected_fn() if xfail_reason is not None: if configuration.results_formatter_object is not None: # Mark this test as expected to fail. configuration.results_formatter_object.handle_event( EventBuilder.event_for_mark_test_expected_failure(self)) xfail_func = unittest2.expectedFailure(func) xfail_func(*args, **kwargs) else: func(*args, **kwargs) return wrapper # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows) # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])). When called # the first way, the first argument will be the actual function because decorators are # weird like that. So this is basically a check that says "which syntax was the original # function decorated with?" if six.callable(bugnumber): return expectedFailure_impl(bugnumber) else: return expectedFailure_impl def skipTestIfFn(expected_fn, bugnumber=None): def skipTestIfFn_impl(func): if isinstance(func, type) and issubclass(func, unittest2.TestCase): raise Exception("@skipTestIfFn can only be used to decorate a test method") @wraps(func) def wrapper(*args, **kwargs): self = args[0] if funcutils.requires_self(expected_fn): reason = expected_fn(self) else: reason = expected_fn() if reason is not None: self.skipTest(reason) else: func(*args, **kwargs) return wrapper # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows) # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])). When called # the first way, the first argument will be the actual function because decorators are # weird like that. So this is basically a check that says "how was the decorator used" if six.callable(bugnumber): return skipTestIfFn_impl(bugnumber) else: return skipTestIfFn_impl def _decorateTest(mode, bugnumber=None, oslist=None, hostoslist=None, compiler=None, compiler_version=None, archs=None, triple=None, debug_info=None, swig_version=None, py_version=None, remote=None): def fn(self): skip_for_os = _match_decorator_property(lldbplatform.translate(oslist), self.getPlatform()) skip_for_hostos = _match_decorator_property(lldbplatform.translate(hostoslist), lldbplatformutil.getHostPlatform()) skip_for_compiler = _match_decorator_property(compiler, self.getCompiler()) and self.expectedCompilerVersion(compiler_version) skip_for_arch = _match_decorator_property(archs, self.getArchitecture()) skip_for_debug_info = _match_decorator_property(debug_info, self.debug_info) skip_for_triple = _match_decorator_property(triple, lldb.DBG.GetSelectedPlatform().GetTriple()) skip_for_remote = _match_decorator_property(remote, lldb.remote_platform is not None) skip_for_swig_version = (swig_version is None) or (not hasattr(lldb, 'swig_version')) or (_check_expected_version(swig_version[0], swig_version[1], lldb.swig_version)) skip_for_py_version = (py_version is None) or _check_expected_version(py_version[0], py_version[1], sys.version_info) # For the test to be skipped, all specified (e.g. not None) parameters must be True. # An unspecified parameter means "any", so those are marked skip by default. And we skip # the final test if all conditions are True. conditions = [(oslist, skip_for_os, "target o/s"), (hostoslist, skip_for_hostos, "host o/s"), (compiler, skip_for_compiler, "compiler or version"), (archs, skip_for_arch, "architecture"), (debug_info, skip_for_debug_info, "debug info format"), (triple, skip_for_triple, "target triple"), (swig_version, skip_for_swig_version, "swig version"), (py_version, skip_for_py_version, "python version"), (remote, skip_for_remote, "platform locality (remote/local)")] reasons = [] final_skip_result = True for this_condition in conditions: final_skip_result = final_skip_result and this_condition[1] if this_condition[0] is not None and this_condition[1]: reasons.append(this_condition[2]) reason_str = None if final_skip_result: mode_str = {DecorateMode.Skip : "skipping", DecorateMode.Xfail : "xfailing"}[mode] if len(reasons) > 0: reason_str = ",".join(reasons) reason_str = "{} due to the following parameter(s): {}".format(mode_str, reason_str) else: reason_str = "{} unconditionally" if bugnumber is not None and not six.callable(bugnumber): reason_str = reason_str + " [" + str(bugnumber) + "]" return reason_str if mode == DecorateMode.Skip: return skipTestIfFn(fn, bugnumber) elif mode == DecorateMode.Xfail: return expectedFailure(fn, bugnumber) else: return None # provide a function to xfail on defined oslist, compiler version, and archs # if none is specified for any argument, that argument won't be checked and thus means for all # for example, # @expectedFailureAll, xfail for all platform/compiler/arch, # @expectedFailureAll(compiler='gcc'), xfail for gcc on all platform/architecture # @expectedFailureAll(bugnumber, ["linux"], "gcc", ['>=', '4.9'], ['i386']), xfail for gcc>=4.9 on linux with i386 def expectedFailureAll(bugnumber=None, oslist=None, hostoslist=None, compiler=None, compiler_version=None, archs=None, triple=None, debug_info=None, swig_version=None, py_version=None, remote=None): return _decorateTest(DecorateMode.Xfail, bugnumber=bugnumber, oslist=oslist, hostoslist=hostoslist, compiler=compiler, compiler_version=compiler_version, archs=archs, triple=triple, debug_info=debug_info, swig_version=swig_version, py_version=py_version, remote=remote) # provide a function to skip on defined oslist, compiler version, and archs # if none is specified for any argument, that argument won't be checked and thus means for all # for example, # @skipIf, skip for all platform/compiler/arch, # @skipIf(compiler='gcc'), skip for gcc on all platform/architecture # @skipIf(bugnumber, ["linux"], "gcc", ['>=', '4.9'], ['i386']), skip for gcc>=4.9 on linux with i386 def skipIf(bugnumber=None, oslist=None, hostoslist=None, compiler=None, compiler_version=None, archs=None, triple=None, debug_info=None, swig_version=None, py_version=None, remote=None): return _decorateTest(DecorateMode.Skip, bugnumber=bugnumber, oslist=oslist, hostoslist=hostoslist, compiler=compiler, compiler_version=compiler_version, archs=archs, triple=triple, debug_info=debug_info, swig_version=swig_version, py_version=py_version, remote=remote) def _skip_for_android(reason, api_levels, archs): def impl(obj): result = lldbplatformutil.match_android_device(obj.getArchitecture(), valid_archs=archs, valid_api_levels=api_levels) return reason if result else None return impl def add_test_categories(cat): """Add test categories to a TestCase method""" cat = test_categories.validate(cat, True) def impl(func): if isinstance(func, type) and issubclass(func, unittest2.TestCase): raise Exception("@add_test_categories can only be used to decorate a test method") if hasattr(func, "categories"): cat.extend(func.categories) func.categories = cat return func return impl def benchmarks_test(func): """Decorate the item as a benchmarks test.""" def should_skip_benchmarks_test(): return "benchmarks test" # Mark this function as such to separate them from the regular tests. result = skipTestIfFn(should_skip_benchmarks_test)(func) result.__benchmarks_test__ = True return result def no_debug_info_test(func): """Decorate the item as a test what don't use any debug info. If this annotation is specified then the test runner won't generate a separate test for each debug info format. """ if isinstance(func, type) and issubclass(func, unittest2.TestCase): raise Exception("@no_debug_info_test can only be used to decorate a test method") @wraps(func) def wrapper(self, *args, **kwargs): return func(self, *args, **kwargs) # Mark this function as such to separate them from the regular tests. wrapper.__no_debug_info_test__ = True return wrapper def debugserver_test(func): """Decorate the item as a debugserver test.""" def should_skip_debugserver_test(): return "debugserver tests" if configuration.dont_do_debugserver_test else None return skipTestIfFn(should_skip_debugserver_test)(func) def llgs_test(func): """Decorate the item as a lldb-server test.""" def should_skip_llgs_tests(): return "llgs tests" if configuration.dont_do_llgs_test else None return skipTestIfFn(should_skip_llgs_tests)(func) def not_remote_testsuite_ready(func): """Decorate the item as a test which is not ready yet for remote testsuite.""" def is_remote(): return "Not ready for remote testsuite" if lldb.remote_platform else None return skipTestIfFn(is_remote)(func) def expectedFailureOS(oslist, bugnumber=None, compilers=None, debug_info=None, archs=None): return expectedFailureAll(oslist=oslist, bugnumber=bugnumber, compiler=compilers, archs=archs, debug_info=debug_info) def expectedFailureDarwin(bugnumber=None, compilers=None, debug_info=None): # For legacy reasons, we support both "darwin" and "macosx" as OS X triples. return expectedFailureOS(lldbplatform.darwin_all, bugnumber, compilers, debug_info=debug_info) def expectedFailureAndroid(bugnumber=None, api_levels=None, archs=None): """ Mark a test as xfail for Android. Arguments: bugnumber - The LLVM pr associated with the problem. api_levels - A sequence of numbers specifying the Android API levels for which a test is expected to fail. None means all API level. arch - A sequence of architecture names specifying the architectures for which a test is expected to fail. None means all architectures. """ return expectedFailure(_skip_for_android("xfailing on android", api_levels, archs), bugnumber) # Flakey tests get two chances to run. If they fail the first time round, the result formatter # makes sure it is run one more time. def expectedFlakey(expected_fn, bugnumber=None): def expectedFailure_impl(func): @wraps(func) def wrapper(*args, **kwargs): self = args[0] if expected_fn(self): # Send event marking test as explicitly eligible for rerunning. if configuration.results_formatter_object is not None: # Mark this test as rerunnable. configuration.results_formatter_object.handle_event( EventBuilder.event_for_mark_test_rerun_eligible(self)) func(*args, **kwargs) return wrapper # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows) # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])). When called # the first way, the first argument will be the actual function because decorators are # weird like that. So this is basically a check that says "which syntax was the original # function decorated with?" if six.callable(bugnumber): return expectedFailure_impl(bugnumber) else: return expectedFailure_impl def expectedFlakeyDwarf(bugnumber=None): def fn(self): return self.debug_info == "dwarf" return expectedFlakey(fn, bugnumber) def expectedFlakeyDsym(bugnumber=None): def fn(self): return self.debug_info == "dwarf" return expectedFlakey(fn, bugnumber) def expectedFlakeyOS(oslist, bugnumber=None, compilers=None): def fn(self): return (self.getPlatform() in oslist and self.expectedCompiler(compilers)) return expectedFlakey(fn, bugnumber) def expectedFlakeyDarwin(bugnumber=None, compilers=None): # For legacy reasons, we support both "darwin" and "macosx" as OS X triples. return expectedFlakeyOS(lldbplatformutil.getDarwinOSTriples(), bugnumber, compilers) def expectedFlakeyFreeBSD(bugnumber=None, compilers=None): return expectedFlakeyOS(['freebsd'], bugnumber, compilers) def expectedFlakeyLinux(bugnumber=None, compilers=None): return expectedFlakeyOS(['linux'], bugnumber, compilers) def expectedFlakeyNetBSD(bugnumber=None, compilers=None): return expectedFlakeyOS(['netbsd'], bugnumber, compilers) def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None): if compiler_version is None: compiler_version=['=', None] def fn(self): return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version) return expectedFlakey(fn, bugnumber) # @expectedFlakeyClang('bugnumber', ['<=', '3.4']) def expectedFlakeyClang(bugnumber=None, compiler_version=None): return expectedFlakeyCompiler('clang', compiler_version, bugnumber) # @expectedFlakeyGcc('bugnumber', ['<=', '3.4']) def expectedFlakeyGcc(bugnumber=None, compiler_version=None): return expectedFlakeyCompiler('gcc', compiler_version, bugnumber) def expectedFlakeyAndroid(bugnumber=None, api_levels=None, archs=None): return expectedFlakey(_skip_for_android("flakey on android", api_levels, archs), bugnumber) def skipIfRemote(func): """Decorate the item to skip tests if testing remotely.""" def is_remote(): return "skip on remote platform" if lldb.remote_platform else None return skipTestIfFn(is_remote)(func) def skipIfRemoteDueToDeadlock(func): """Decorate the item to skip tests if testing remotely due to the test deadlocking.""" def is_remote(): return "skip on remote platform (deadlocks)" if lldb.remote_platform else None return skipTestIfFn(is_remote)(func) def skipIfNoSBHeaders(func): """Decorate the item to mark tests that should be skipped when LLDB is built with no SB API headers.""" def are_sb_headers_missing(): if lldbplatformutil.getHostPlatform() == 'darwin': header = os.path.join(os.environ["LLDB_LIB_DIR"], 'LLDB.framework', 'Versions','Current','Headers','LLDB.h') else: header = os.path.join(os.environ["LLDB_SRC"], "include", "lldb", "API", "LLDB.h") if not os.path.exists(header): return "skip because LLDB.h header not found" return None return skipTestIfFn(are_sb_headers_missing)(func) def skipIfiOSSimulator(func): """Decorate the item to skip tests that should be skipped on the iOS Simulator.""" def is_ios_simulator(): return "skip on the iOS Simulator" if configuration.lldb_platform_name == 'ios-simulator' else None return skipTestIfFn(is_ios_simulator)(func) def skipIfFreeBSD(func): """Decorate the item to skip tests that should be skipped on FreeBSD.""" return skipIfPlatform(["freebsd"])(func) def skipIfNetBSD(func): """Decorate the item to skip tests that should be skipped on NetBSD.""" return skipIfPlatform(["netbsd"])(func) def skipIfDarwin(func): """Decorate the item to skip tests that should be skipped on Darwin.""" return skipIfPlatform(lldbplatform.translate(lldbplatform.darwin_all))(func) def skipIfLinux(func): """Decorate the item to skip tests that should be skipped on Linux.""" return skipIfPlatform(["linux"])(func) def skipIfWindows(func): """Decorate the item to skip tests that should be skipped on Windows.""" return skipIfPlatform(["windows"])(func) def skipUnlessWindows(func): """Decorate the item to skip tests that should be skipped on any non-Windows platform.""" return skipUnlessPlatform(["windows"])(func) def skipUnlessDarwin(func): """Decorate the item to skip tests that should be skipped on any non Darwin platform.""" return skipUnlessPlatform(lldbplatformutil.getDarwinOSTriples())(func) def skipUnlessGoInstalled(func): """Decorate the item to skip tests when no Go compiler is available.""" def is_go_missing(self): compiler = self.getGoCompilerVersion() if not compiler: return "skipping because go compiler not found" match_version = re.search(r"(\d+\.\d+(\.\d+)?)", compiler) if not match_version: # Couldn't determine version. return "skipping because go version could not be parsed out of {}".format(compiler) else: min_strict_version = StrictVersion("1.4.0") compiler_strict_version = StrictVersion(match_version.group(1)) if compiler_strict_version < min_strict_version: return "skipping because available version ({}) does not meet minimum required version ({})".format( compiler_strict_version, min_strict_version) return None return skipTestIfFn(is_go_missing)(func) def skipIfHostIncompatibleWithRemote(func): """Decorate the item to skip tests if binaries built on this host are incompatible.""" def is_host_incompatible_with_remote(self): host_arch = self.getLldbArchitecture() host_platform = lldbplatformutil.getHostPlatform() target_arch = self.getArchitecture() target_platform = 'darwin' if self.platformIsDarwin() else self.getPlatform() if not (target_arch == 'x86_64' and host_arch == 'i386') and host_arch != target_arch: return "skipping because target %s is not compatible with host architecture %s" % (target_arch, host_arch) elif target_platform != host_platform: return "skipping because target is %s but host is %s" % (target_platform, host_platform) return None return skipTestIfFn(is_host_incompatible_with_remote)(func) def skipIfPlatform(oslist): """Decorate the item to skip tests if running on one of the listed platforms.""" # This decorator cannot be ported to `skipIf` yet because it is used on entire # classes, which `skipIf` explicitly forbids. return unittest2.skipIf(lldbplatformutil.getPlatform() in oslist, "skip on %s" % (", ".join(oslist))) def skipUnlessPlatform(oslist): """Decorate the item to skip tests unless running on one of the listed platforms.""" # This decorator cannot be ported to `skipIf` yet because it is used on entire # classes, which `skipIf` explicitly forbids. return unittest2.skipUnless(lldbplatformutil.getPlatform() in oslist, "requires on of %s" % (", ".join(oslist))) def skipIfTargetAndroid(api_levels=None, archs=None): """Decorator to skip tests when the target is Android. Arguments: api_levels - The API levels for which the test should be skipped. If it is None, then the test will be skipped for all API levels. arch - A sequence of architecture names specifying the architectures for which a test is skipped. None means all architectures. """ return skipTestIfFn(_skip_for_android("skipping for android", api_levels, archs)) def skipUnlessCompilerRt(func): """Decorate the item to skip tests if testing remotely.""" def is_compiler_rt_missing(): compilerRtPath = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "llvm","projects","compiler-rt") return "compiler-rt not found" if not os.path.exists(compilerRtPath) else None return skipTestIfFn(is_compiler_rt_missing)(func) def skipUnlessThreadSanitizer(func): """Decorate the item to skip test unless Clang -fsanitize=thread is supported.""" def is_compiler_clang_with_thread_sanitizer(self): compiler_path = self.getCompiler() compiler = os.path.basename(compiler_path) if not compiler.startswith("clang"): return "Test requires clang as compiler" f = tempfile.NamedTemporaryFile() cmd = "echo 'int main() {}' | %s -x c -o %s -" % (compiler_path, f.name) if os.popen(cmd).close() != None: return None # The compiler cannot compile at all, let's *not* skip the test cmd = "echo 'int main() {}' | %s -fsanitize=thread -x c -o %s -" % (compiler_path, f.name) if os.popen(cmd).close() != None: return "Compiler cannot compile with -fsanitize=thread" return None return skipTestIfFn(is_compiler_clang_with_thread_sanitizer)(func)