aboutsummaryrefslogtreecommitdiff
path: root/utils/lit/tests
diff options
context:
space:
mode:
Diffstat (limited to 'utils/lit/tests')
-rw-r--r--utils/lit/tests/Inputs/shtest-format/requires-missing.txt7
-rw-r--r--utils/lit/tests/Inputs/shtest-format/requires-present.txt4
-rw-r--r--utils/lit/tests/Inputs/shtest-format/requires-star.txt3
-rw-r--r--utils/lit/tests/Inputs/shtest-format/requires-triple.txt3
-rw-r--r--utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt9
-rw-r--r--utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt4
-rw-r--r--utils/lit/tests/Inputs/shtest-format/unsupported-star.txt3
-rw-r--r--utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt3
-rw-r--r--utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt4
-rw-r--r--utils/lit/tests/Inputs/test-data/dummy_format.py38
-rw-r--r--utils/lit/tests/Inputs/test-data/lit.cfg42
-rw-r--r--utils/lit/tests/boolean-parsing.py4
-rw-r--r--utils/lit/tests/selecting.py90
-rw-r--r--utils/lit/tests/shtest-format.py15
-rw-r--r--utils/lit/tests/unit/TestRunner.py61
15 files changed, 243 insertions, 47 deletions
diff --git a/utils/lit/tests/Inputs/shtest-format/requires-missing.txt b/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
index 9e6648d8b8f0..d643e57edcad 100644
--- a/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
+++ b/utils/lit/tests/Inputs/shtest-format/requires-missing.txt
@@ -1,2 +1,5 @@
-RUN: true
-REQUIRES: a-missing-feature
+# REQUIRES with a false clause. Test should not run.
+REQUIRES: true
+REQUIRES: a-missing-feature, true
+REQUIRES: true
+RUN: false
diff --git a/utils/lit/tests/Inputs/shtest-format/requires-present.txt b/utils/lit/tests/Inputs/shtest-format/requires-present.txt
index 064f7074a76e..9fcbdca69be3 100644
--- a/utils/lit/tests/Inputs/shtest-format/requires-present.txt
+++ b/utils/lit/tests/Inputs/shtest-format/requires-present.txt
@@ -1,2 +1,4 @@
+# REQUIRES with only true clauses. Test should run.
+REQUIRES: a-present-feature, true, !not-true
+REQUIRES: true
RUN: true
-REQUIRES: a-present-feature
diff --git a/utils/lit/tests/Inputs/shtest-format/requires-star.txt b/utils/lit/tests/Inputs/shtest-format/requires-star.txt
new file mode 100644
index 000000000000..5566d8b15b07
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/requires-star.txt
@@ -0,0 +1,3 @@
+# '*' only works in XFAIL
+REQUIRES: *
+RUN: false
diff --git a/utils/lit/tests/Inputs/shtest-format/requires-triple.txt b/utils/lit/tests/Inputs/shtest-format/requires-triple.txt
new file mode 100644
index 000000000000..6470bf404145
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/requires-triple.txt
@@ -0,0 +1,3 @@
+# REQUIRES line that uses target triple, which doesn't work. Test should not run
+REQUIRES: x86_64
+RUN: false
diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
new file mode 100644
index 000000000000..00c6160a367c
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
@@ -0,0 +1,9 @@
+# UNSUPPORTED with only false clauses. Test should run.
+UNSUPPORTED: false
+UNSUPPORTED: false, not-true
+UNSUPPORTED: false
+UNSUPPORTED: still-not-true
+UNSUPPORTED: false
+UNSUPPORTED: false
+UNSUPPORTED: false
+RUN: true
diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
new file mode 100644
index 000000000000..f48ba7b2c2d2
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
@@ -0,0 +1,4 @@
+# UNSUPPORTED with a true clause. Test should not run.
+UNSUPPORTED: false
+UNSUPPORTED: false, false, false, _64-unk && a-present-feature, false
+RUN: false
diff --git a/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt b/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt
new file mode 100644
index 000000000000..16630207dacb
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/unsupported-star.txt
@@ -0,0 +1,3 @@
+# '*' only works in XFAIL
+UNSUPPORTED: *
+RUN: false
diff --git a/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt b/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
new file mode 100644
index 000000000000..83b0de1621d0
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
@@ -0,0 +1,3 @@
+# XFAIL with only false clauses. Test should run.
+XFAIL: false, a-missing-feature || ! a-present-feature || ! x86_64, false
+RUN: true
diff --git a/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt b/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
new file mode 100644
index 000000000000..3c197484897e
--- /dev/null
+++ b/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
@@ -0,0 +1,4 @@
+# XFAIL with a true clause. Test should not run.
+XFAIL: false
+XFAIL: false, a-present-feature && ! a-missing-feature && x86_64
+RUN: false
diff --git a/utils/lit/tests/Inputs/test-data/dummy_format.py b/utils/lit/tests/Inputs/test-data/dummy_format.py
new file mode 100644
index 000000000000..93e48eeb8396
--- /dev/null
+++ b/utils/lit/tests/Inputs/test-data/dummy_format.py
@@ -0,0 +1,38 @@
+import os
+try:
+ import ConfigParser
+except ImportError:
+ import configparser as ConfigParser
+
+import lit.formats
+import lit.Test
+
+class DummyFormat(lit.formats.FileBasedTest):
+ def execute(self, test, lit_config):
+ # In this dummy format, expect that each test file is actually just a
+ # .ini format dump of the results to report.
+
+ source_path = test.getSourcePath()
+
+ cfg = ConfigParser.ConfigParser()
+ cfg.read(source_path)
+
+ # Create the basic test result.
+ result_code = cfg.get('global', 'result_code')
+ result_output = cfg.get('global', 'result_output')
+ result = lit.Test.Result(getattr(lit.Test, result_code),
+ result_output)
+
+ # Load additional metrics.
+ for key,value_str in cfg.items('results'):
+ value = eval(value_str)
+ if isinstance(value, int):
+ metric = lit.Test.IntMetricValue(value)
+ elif isinstance(value, float):
+ metric = lit.Test.RealMetricValue(value)
+ else:
+ raise RuntimeError("unsupported result type")
+ result.addMetric(key, metric)
+
+ return result
+
diff --git a/utils/lit/tests/Inputs/test-data/lit.cfg b/utils/lit/tests/Inputs/test-data/lit.cfg
index f5aba7b21774..0191cc218884 100644
--- a/utils/lit/tests/Inputs/test-data/lit.cfg
+++ b/utils/lit/tests/Inputs/test-data/lit.cfg
@@ -1,44 +1,10 @@
-import os
-try:
- import ConfigParser
-except ImportError:
- import configparser as ConfigParser
-
-import lit.formats
-import lit.Test
-
-class DummyFormat(lit.formats.FileBasedTest):
- def execute(self, test, lit_config):
- # In this dummy format, expect that each test file is actually just a
- # .ini format dump of the results to report.
-
- source_path = test.getSourcePath()
-
- cfg = ConfigParser.ConfigParser()
- cfg.read(source_path)
-
- # Create the basic test result.
- result_code = cfg.get('global', 'result_code')
- result_output = cfg.get('global', 'result_output')
- result = lit.Test.Result(getattr(lit.Test, result_code),
- result_output)
-
- # Load additional metrics.
- for key,value_str in cfg.items('results'):
- value = eval(value_str)
- if isinstance(value, int):
- metric = lit.Test.IntMetricValue(value)
- elif isinstance(value, float):
- metric = lit.Test.RealMetricValue(value)
- else:
- raise RuntimeError("unsupported result type")
- result.addMetric(key, metric)
-
- return result
+import site
+site.addsitedir(os.path.dirname(__file__))
+import dummy_format
config.name = 'test-data'
config.suffixes = ['.ini']
-config.test_format = DummyFormat()
+config.test_format = dummy_format.DummyFormat()
config.test_source_root = None
config.test_exec_root = None
config.target_triple = None
diff --git a/utils/lit/tests/boolean-parsing.py b/utils/lit/tests/boolean-parsing.py
new file mode 100644
index 000000000000..372a94d23323
--- /dev/null
+++ b/utils/lit/tests/boolean-parsing.py
@@ -0,0 +1,4 @@
+# Test the boolean expression parser
+# used for REQUIRES and UNSUPPORTED and XFAIL
+
+# RUN: %{python} -m lit.BooleanExpression
diff --git a/utils/lit/tests/selecting.py b/utils/lit/tests/selecting.py
new file mode 100644
index 000000000000..72d6fbabdc93
--- /dev/null
+++ b/utils/lit/tests/selecting.py
@@ -0,0 +1,90 @@
+# RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s
+# CHECK-BASIC: Testing: 5 tests
+
+
+# Check that regex-filtering works
+#
+# RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
+# CHECK-FILTER: Testing: 2 of 5 tests
+
+
+# Check that maximum counts work
+#
+# RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
+# CHECK-MAX: Testing: 3 of 5 tests
+
+
+# Check that sharding partitions the testsuite in a way that distributes the
+# rounding error nicely (i.e. 5/3 => 2 2 1, not 1 1 3 or whatever)
+#
+# RUN: %{lit} --num-shards 3 --run-shard 1 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD0-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD0-OUT < %t.out %s
+# CHECK-SHARD0-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
+# CHECK-SHARD0-OUT: Testing: 2 of 5 tests
+#
+# RUN: %{lit} --num-shards 3 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD1-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD1-OUT < %t.out %s
+# CHECK-SHARD1-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
+# CHECK-SHARD1-OUT: Testing: 2 of 5 tests
+#
+# RUN: %{lit} --num-shards 3 --run-shard 3 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD2-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD2-OUT < %t.out %s
+# CHECK-SHARD2-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
+# CHECK-SHARD2-OUT: Testing: 1 of 5 tests
+
+
+# Check that sharding via env vars works.
+#
+# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=1 %{lit} %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-OUT < %t.out %s
+# CHECK-SHARD0-ENV-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
+# CHECK-SHARD0-ENV-OUT: Testing: 2 of 5 tests
+#
+# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=2 %{lit} %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-OUT < %t.out %s
+# CHECK-SHARD1-ENV-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
+# CHECK-SHARD1-ENV-OUT: Testing: 2 of 5 tests
+#
+# RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=3 %{lit} %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-ERR < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-OUT < %t.out %s
+# CHECK-SHARD2-ENV-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
+# CHECK-SHARD2-ENV-OUT: Testing: 1 of 5 tests
+
+
+# Check that providing more shards than tests results in 1 test per shard
+# until we run out, then 0.
+#
+# RUN: %{lit} --num-shards 100 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR1 < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT1 < %t.out %s
+# CHECK-SHARD-BIG-ERR1: note: Selecting shard 2/100 = size 1/5 = tests #(100*k)+2 = [2]
+# CHECK-SHARD-BIG-OUT1: Testing: 1 of 5 tests
+#
+# RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s
+# CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []
+# CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests
+#
+# RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s
+# CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []
+# CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests
+
+
+# Check that range constraints are enforced
+#
+# RUN: not %{lit} --num-shards 0 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD-ERR < %t.err %s
+# CHECK-SHARD-ERR: error: --num-shards must be positive
+#
+# RUN: not %{lit} --num-shards 3 --run-shard 4 %{inputs}/discovery >%t.out 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-SHARD-ERR2 < %t.err %s
+# CHECK-SHARD-ERR2: error: --run-shard must be between 1 and --num-shards (inclusive)
diff --git a/utils/lit/tests/shtest-format.py b/utils/lit/tests/shtest-format.py
index 20884f8c4854..37e3e1c12629 100644
--- a/utils/lit/tests/shtest-format.py
+++ b/utils/lit/tests/shtest-format.py
@@ -50,7 +50,14 @@
# CHECK: PASS: shtest-format :: requires-any-present.txt
# CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
# CHECK: PASS: shtest-format :: requires-present.txt
+# CHECK: UNRESOLVED: shtest-format :: requires-star.txt
+# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
+# CHECK: PASS: shtest-format :: unsupported-expr-false.txt
+# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
+# CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
# CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
+# CHECK: PASS: shtest-format :: xfail-expr-false.txt
+# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
# CHECK: XFAIL: shtest-format :: xfail-feature.txt
# CHECK: XFAIL: shtest-format :: xfail-target.txt
# CHECK: XFAIL: shtest-format :: xfail.txt
@@ -70,9 +77,9 @@
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
# CHECK: shtest-format :: fail.txt
-# CHECK: Expected Passes : 5
-# CHECK: Expected Failures : 3
-# CHECK: Unsupported Tests : 3
-# CHECK: Unresolved Tests : 1
+# CHECK: Expected Passes : 7
+# CHECK: Expected Failures : 4
+# CHECK: Unsupported Tests : 5
+# CHECK: Unresolved Tests : 3
# CHECK: Unexpected Passes : 1
# CHECK: Unexpected Failures: 3
diff --git a/utils/lit/tests/unit/TestRunner.py b/utils/lit/tests/unit/TestRunner.py
index ff11834fed7e..79cc10f7e14d 100644
--- a/utils/lit/tests/unit/TestRunner.py
+++ b/utils/lit/tests/unit/TestRunner.py
@@ -89,7 +89,7 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
parsers = self.make_parsers()
self.parse_test(parsers)
list_parser = self.get_parser(parsers, 'MY_LIST:')
- self.assertItemsEqual(list_parser.getValue(),
+ self.assertEqual(list_parser.getValue(),
['one', 'two', 'three', 'four'])
def test_commands(self):
@@ -106,8 +106,65 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
self.parse_test(parsers)
custom_parser = self.get_parser(parsers, 'MY_CUSTOM:')
value = custom_parser.getValue()
- self.assertItemsEqual(value, ['a', 'b', 'c'])
+ self.assertEqual(value, ['a', 'b', 'c'])
+ def test_bad_keywords(self):
+ def custom_parse(line_number, line, output):
+ return output
+
+ try:
+ IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
+ self.fail("TAG_NO_SUFFIX failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
+ self.fail("TAG_WITH_COLON: failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
+ self.fail("LIST_WITH_DOT. failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("CUSTOM_NO_SUFFIX",
+ ParserKind.CUSTOM, custom_parse),
+ self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)
+
+ # Both '.' and ':' are allowed for CUSTOM keywords.
+ try:
+ IntegratedTestKeywordParser("CUSTOM_WITH_DOT.",
+ ParserKind.CUSTOM, custom_parse),
+ except BaseException as e:
+ self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
+ try:
+ IntegratedTestKeywordParser("CUSTOM_WITH_COLON:",
+ ParserKind.CUSTOM, custom_parse),
+ except BaseException as e:
+ self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)
+
+ try:
+ IntegratedTestKeywordParser("CUSTOM_NO_PARSER:",
+ ParserKind.CUSTOM),
+ self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
+ except ValueError as e:
+ pass
+ except BaseException as e:
+ self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
if __name__ == '__main__':
TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-15 17:27:44 +0000