diff options
Diffstat (limited to 'packages/Python/lldbsuite/test/issue_verification')
14 files changed, 254 insertions, 0 deletions
diff --git a/packages/Python/lldbsuite/test/issue_verification/Makefile b/packages/Python/lldbsuite/test/issue_verification/Makefile new file mode 100644 index 0000000000000..e7bd3f4dd7924 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/Makefile @@ -0,0 +1,4 @@ +LEVEL = ../make +CXX_SOURCES := inline_rerun_inferior.cpp +CXXFLAGS += -std=c++11 +include $(LEVEL)/Makefile.rules diff --git a/packages/Python/lldbsuite/test/issue_verification/README.txt b/packages/Python/lldbsuite/test/issue_verification/README.txt new file mode 100644 index 0000000000000..0f1ae7f0ecfc4 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/README.txt @@ -0,0 +1,5 @@ +Tests in this directory are intentionally setup to +fail, error, timeout, etc. to verify that the buildbots +pick up errors. The tests in this directory will be +parked/removed/renamed after verifying they trigger +as expected. diff --git a/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park b/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park new file mode 100644 index 0000000000000..67db8149f8504 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park @@ -0,0 +1,20 @@ +"""Tests that a timeout is detected by the testbot.""" +from __future__ import print_function + +import time + +import lldbsuite.test.lldbtest as lldbtest + + +class ExpectedTimeoutTestCase(lldbtest.TestBase): + """Forces test timeout.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @lldbtest.expectedFailureAll() + def test_buildbot_sees_expected_timeout(self): + """Tests that expected timeout logic kicks in and is picked up.""" + while True: + try: + time.sleep(1) + except: + print("ignoring exception during sleep") diff --git a/packages/Python/lldbsuite/test/issue_verification/TestFail.py.park b/packages/Python/lldbsuite/test/issue_verification/TestFail.py.park new file mode 100644 index 0000000000000..da64bc0ea8c17 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestFail.py.park @@ -0,0 +1,16 @@ +"""Tests that a FAIL is detected by the testbot.""" + +from __future__ import print_function + +import lldbsuite.test.lldbtest as lldbtest + + +class FailTestCase(lldbtest.TestBase): + """Forces test failure.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + def test_buildbot_catches_failure(self): + """Issues a failing test assertion.""" + self.assertTrue( + False, + "This will always fail, buildbot should flag this.") diff --git a/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park b/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park new file mode 100644 index 0000000000000..bcd1926d74047 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park @@ -0,0 +1,23 @@ +"""Tests that a flakey fail is rerun, and will pass on the rerun. +Run this test with --rerun-all-issues specified to test that +the tests fail on the first run, then pass on the second. +Do not mark them as flakey as, at this time, flakey tests will +run twice, thus causing the second run to succeed.""" + +from __future__ import print_function + +import rerun_base + +import lldbsuite.test.lldbtest as lldbtest + + +class RerunFailTestCase(rerun_base.RerunBaseTestCase): + """Forces test failure on first run, success on rerun.""" + @lldbtest.no_debug_info_test + def test_buildbot_catches_failure(self): + """Issues a failing test assertion.""" + if self.should_generate_issue(): + self.assertTrue( + False, + "This will fail on the first call, succeed on rerun, and " + "alternate thereafter.") diff --git a/packages/Python/lldbsuite/test/issue_verification/TestRerunInline.py.park b/packages/Python/lldbsuite/test/issue_verification/TestRerunInline.py.park new file mode 100644 index 0000000000000..4c50495a2ec4a --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestRerunInline.py.park @@ -0,0 +1,13 @@ +"""Tests that the rerun mechanism respects lldbinline-created tests. + +The current implementation of this test is expected to fail both on +the initial run and on the rerun, assuming --rerun-all-issues is provided +to the dotest.py run. + +This test could be improved by doing something in the test inferior +C++ program that could look for the "should an issue be raised" marker +file, and then really pass on the rerun. +""" +import lldbsuite.test.lldbinline as lldbinline + +lldbinline.MakeInlineTest(__file__, globals()) diff --git a/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park b/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park new file mode 100644 index 0000000000000..1cf5373ac4916 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park @@ -0,0 +1,22 @@ +"""Tests that a timeout is detected by the testbot.""" +from __future__ import print_function + +import time + +import lldbsuite.test.lldbtest as lldbtest +import rerun_base + + +class RerunTimeoutTestCase(rerun_base.RerunBaseTestCase): + @lldbtest.no_debug_info_test + def test_timeout_rerun_succeeds(self): + """Tests that timeout logic kicks in and is picked up.""" + if not self.should_generate_issue(): + # We pass this time. + return + # We time out this time. + while True: + try: + time.sleep(1) + except: + print("ignoring exception during sleep") diff --git a/packages/Python/lldbsuite/test/issue_verification/TestSignal.py.park b/packages/Python/lldbsuite/test/issue_verification/TestSignal.py.park new file mode 100644 index 0000000000000..d73ac74b47b23 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestSignal.py.park @@ -0,0 +1,26 @@ +"""Tests that an exceptional exit is detected by the testbot.""" + +from __future__ import print_function + +import os +import signal +import time + +import lldbsuite.test.lldbtest as lldbtest + + +class ExceptionalExitTestCase(lldbtest.TestBase): + """Forces exceptional exit.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @lldbtest.skipIfWindows + def test_buildbot_catches_exceptional_exit(self): + """Force process to die with exceptional exit.""" + + # Sleep for a couple seconds + try: + time.sleep(5) + except: + pass + + os.kill(os.getpid(), signal.SIGKILL) diff --git a/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park b/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park new file mode 100644 index 0000000000000..7a5b2ba99f4b1 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park @@ -0,0 +1,24 @@ +"""Tests that an exceptional exit is detected by the testbot.""" + +from __future__ import print_function + +import atexit +import os +import signal +import time + +import lldbsuite.test.lldbtest as lldbtest + + +class ExceptionalExitOutOfTestMethodTestCase(lldbtest.TestBase): + """Forces exceptional exit.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @lldbtest.skipIfWindows + def test_buildbot_catches_exceptional_exit(self): + pass + +def cleanup(): + os.kill(os.getpid(), signal.SIGKILL) + +atexit.register(cleanup) diff --git a/packages/Python/lldbsuite/test/issue_verification/TestTimeout.py.park b/packages/Python/lldbsuite/test/issue_verification/TestTimeout.py.park new file mode 100644 index 0000000000000..ba7be454f9a9a --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/TestTimeout.py.park @@ -0,0 +1,19 @@ +"""Tests that a timeout is detected by the testbot.""" +from __future__ import print_function + +import time + +import lldbsuite.test.lldbtest as lldbtest + + +class TimeoutTestCase(lldbtest.TestBase): + """Forces test timeout.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + def test_buildbot_catches_timeout(self): + """Tests that timeout logic kicks in and is picked up.""" + while True: + try: + time.sleep(1) + except: + print("ignoring exception during sleep") diff --git a/packages/Python/lldbsuite/test/issue_verification/disable.py b/packages/Python/lldbsuite/test/issue_verification/disable.py new file mode 100755 index 0000000000000..6d1f93e8b15dd --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/disable.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +"""Renames *.py files to *.py.park.""" +import os +import sys + + +def main(): + """Drives the main script behavior.""" + script_dir = os.path.dirname(os.path.realpath(__file__)) + for filename in os.listdir(script_dir): + basename, extension = os.path.splitext(filename) + if basename.startswith("Test") and extension == '.py': + source_path = os.path.join(script_dir, filename) + dest_path = source_path + ".park" + sys.stdout.write("renaming {} to {}\n".format( + source_path, dest_path)) + os.rename(source_path, dest_path) + +if __name__ == "__main__": + main() diff --git a/packages/Python/lldbsuite/test/issue_verification/enable.py b/packages/Python/lldbsuite/test/issue_verification/enable.py new file mode 100755 index 0000000000000..eb19276de1f7a --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/enable.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +"""Renames *.py.park files to *.py.""" +import os +import sys + + +def main(): + """Drives the main script behavior.""" + script_dir = os.path.dirname(os.path.realpath(__file__)) + for filename in os.listdir(script_dir): + basename, extension = os.path.splitext(filename) + if basename.startswith("Test") and extension == '.park': + source_path = os.path.join(script_dir, filename) + dest_path = os.path.join(script_dir, basename) + sys.stdout.write("renaming {} to {}\n".format( + source_path, dest_path)) + os.rename(source_path, dest_path) + +if __name__ == "__main__": + main() diff --git a/packages/Python/lldbsuite/test/issue_verification/inline_rerun_inferior.cpp b/packages/Python/lldbsuite/test/issue_verification/inline_rerun_inferior.cpp new file mode 100644 index 0000000000000..933911f7b28d6 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/inline_rerun_inferior.cpp @@ -0,0 +1,14 @@ +//===-- main.cpp --------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +typedef int Foo; + +int main() { + Foo array[3] = {1,2,3}; + return 0; //% self.expect("frame variable array --show-types --", substrs = ['(Foo [3]) wrong_type_here = {','(Foo) [0] = 1','(Foo) [1] = 2','(Foo) [2] = 3']) +} diff --git a/packages/Python/lldbsuite/test/issue_verification/rerun_base.py b/packages/Python/lldbsuite/test/issue_verification/rerun_base.py new file mode 100644 index 0000000000000..2ce775dced139 --- /dev/null +++ b/packages/Python/lldbsuite/test/issue_verification/rerun_base.py @@ -0,0 +1,28 @@ +from __future__ import print_function + +import os + +import lldbsuite.test.lldbtest as lldbtest + + +# pylint: disable=too-few-public-methods +class RerunBaseTestCase(lldbtest.TestBase): + """Forces test failure.""" + mydir = lldbtest.TestBase.compute_mydir(__file__) + + def should_generate_issue(self): + """Returns whether a test issue should be generated. + + @returns True on the first and every other call via a given + test method. + """ + should_pass_filename = "{}.{}.succeed-marker".format( + __file__, self.id()) + fail = not os.path.exists(should_pass_filename) + if fail: + # Create the marker so that next call to this passes. + open(should_pass_filename, 'w').close() + else: + # Delete the marker so next time we fail. + os.remove(should_pass_filename) + return fail |