summaryrefslogtreecommitdiff
path: root/utils/lit
diff options
context:
space:
mode:
Diffstat (limited to 'utils/lit')
-rw-r--r--utils/lit/lit/ProgressBar.py8
-rw-r--r--utils/lit/lit/Test.py54
-rw-r--r--utils/lit/lit/TestRunner.py98
-rw-r--r--utils/lit/lit/TestingConfig.py14
-rw-r--r--utils/lit/lit/__init__.py2
-rw-r--r--utils/lit/lit/discovery.py2
-rw-r--r--utils/lit/lit/formats/googletest.py1
-rwxr-xr-xutils/lit/lit/main.py16
-rw-r--r--utils/lit/lit/util.py27
-rw-r--r--utils/lit/tests/test-output.py2
-rw-r--r--utils/lit/tests/xunit-output.py10
11 files changed, 168 insertions, 66 deletions
diff --git a/utils/lit/lit/ProgressBar.py b/utils/lit/lit/ProgressBar.py
index e3644f1fa634..3ad704d16c92 100644
--- a/utils/lit/lit/ProgressBar.py
+++ b/utils/lit/lit/ProgressBar.py
@@ -6,8 +6,8 @@
import sys, re, time
def to_bytes(str):
- # Encode to Latin1 to get binary data.
- return str.encode('ISO-8859-1')
+ # Encode to UTF-8 to get binary data.
+ return str.encode('utf-8')
class TerminalController:
"""
@@ -136,7 +136,7 @@ class TerminalController:
def _tparm(self, arg, index):
import curses
- return curses.tparm(to_bytes(arg), index).decode('ascii') or ''
+ return curses.tparm(to_bytes(arg), index).decode('utf-8') or ''
def _tigetstr(self, cap_name):
# String capabilities can include "delays" of the form "$<2>".
@@ -147,7 +147,7 @@ class TerminalController:
if cap is None:
cap = ''
else:
- cap = cap.decode('ascii')
+ cap = cap.decode('utf-8')
return re.sub(r'\$<\d+>[/*]?', '', cap)
def render(self, template):
diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py
index 2e0f478337bc..b81023010d70 100644
--- a/utils/lit/lit/Test.py
+++ b/utils/lit/lit/Test.py
@@ -1,5 +1,6 @@
import os
from xml.sax.saxutils import escape
+from json import JSONEncoder
# Test result codes.
@@ -73,6 +74,41 @@ class RealMetricValue(MetricValue):
def todata(self):
return self.value
+class JSONMetricValue(MetricValue):
+ """
+ JSONMetricValue is used for types that are representable in the output
+ but that are otherwise uninterpreted.
+ """
+ def __init__(self, value):
+ # Ensure the value is a serializable by trying to encode it.
+ # WARNING: The value may change before it is encoded again, and may
+ # not be encodable after the change.
+ try:
+ e = JSONEncoder()
+ e.encode(value)
+ except TypeError:
+ raise
+ self.value = value
+
+ def format(self):
+ return str(self.value)
+
+ def todata(self):
+ return self.value
+
+def toMetricValue(value):
+ if isinstance(value, MetricValue):
+ return value
+ elif isinstance(value, int) or isinstance(value, long):
+ return IntMetricValue(value)
+ elif isinstance(value, float):
+ return RealMetricValue(value)
+ else:
+ # Try to create a JSONMetricValue and let the constructor throw
+ # if value is not a valid type.
+ return JSONMetricValue(value)
+
+
# Test results.
class Result(object):
@@ -200,12 +236,20 @@ class Test:
def getJUnitXML(self):
test_name = self.path_in_suite[-1]
test_path = self.path_in_suite[:-1]
-
- xml = "<testcase classname='" + self.suite.name + "." + "/".join(test_path) + "'" + " name='" + test_name + "'"
+ safe_test_path = [x.replace(".","_") for x in test_path]
+ safe_name = self.suite.name.replace(".","-")
+
+ if safe_test_path:
+ class_name = safe_name + "." + "/".join(safe_test_path)
+ else:
+ class_name = safe_name + "." + safe_name
+
+ xml = "<testcase classname='" + class_name + "' name='" + \
+ test_name + "'"
xml += " time='%.2f'" % (self.result.elapsed,)
if self.result.code.isFailure:
- xml += ">\n\t<failure >\n" + escape(self.result.output)
- xml += "\n\t</failure>\n</testcase>"
+ xml += ">\n\t<failure >\n" + escape(self.result.output)
+ xml += "\n\t</failure>\n</testcase>"
else:
- xml += "/>"
+ xml += "/>"
return xml \ No newline at end of file
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index 97524179988d..268e46c38f74 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -7,6 +7,7 @@ import tempfile
import lit.ShUtil as ShUtil
import lit.Test as Test
import lit.util
+from lit.util import to_bytes, to_string
class InternalShellError(Exception):
def __init__(self, command, message):
@@ -144,13 +145,16 @@ def executeShCmd(cmd, cfg, cwd, results):
named_temp_files.append(f.name)
args[i] = f.name
- procs.append(subprocess.Popen(args, cwd=cwd,
- executable = executable,
- stdin = stdin,
- stdout = stdout,
- stderr = stderr,
- env = cfg.environment,
- close_fds = kUseCloseFDs))
+ try:
+ procs.append(subprocess.Popen(args, cwd=cwd,
+ executable = executable,
+ stdin = stdin,
+ stdout = stdout,
+ stderr = stderr,
+ env = cfg.environment,
+ close_fds = kUseCloseFDs))
+ except OSError as e:
+ raise InternalShellError(j, 'Could not create process due to {}'.format(e))
# Immediately close stdin for any process taking stdin from us.
if stdin == subprocess.PIPE:
@@ -192,6 +196,11 @@ def executeShCmd(cmd, cfg, cwd, results):
f.seek(0, 0)
procData[i] = (procData[i][0], f.read())
+ def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return bytes.encode('utf-8')
+
exitCode = None
for i,(out,err) in enumerate(procData):
res = procs[i].wait()
@@ -201,11 +210,11 @@ def executeShCmd(cmd, cfg, cwd, results):
# Ensure the resulting output is always of string type.
try:
- out = str(out.decode('ascii'))
+ out = to_string(out.decode('utf-8'))
except:
out = str(out)
try:
- err = str(err.decode('ascii'))
+ err = to_string(err.decode('utf-8'))
except:
err = str(err)
@@ -314,14 +323,11 @@ def parseIntegratedTestScriptCommands(source_path):
# Python2 and bytes in Python3.
#
# Once we find a match, we do require each script line to be decodable to
- # ascii, so we convert the outputs to ascii before returning. This way the
+ # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
# remaining code can work with "strings" agnostic of the executing Python
# version.
-
- def to_bytes(str):
- # Encode to Latin1 to get binary data.
- return str.encode('ISO-8859-1')
- keywords = ('RUN:', 'XFAIL:', 'REQUIRES:', 'END.')
+
+ keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
keywords_re = re.compile(
to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
@@ -330,6 +336,10 @@ def parseIntegratedTestScriptCommands(source_path):
# Read the entire file contents.
data = f.read()
+ # Ensure the data ends with a newline.
+ if not data.endswith(to_bytes('\n')):
+ data = data + to_bytes('\n')
+
# Iterate over the matches.
line_number = 1
last_match_position = 0
@@ -341,21 +351,25 @@ def parseIntegratedTestScriptCommands(source_path):
match_position)
last_match_position = match_position
- # Convert the keyword and line to ascii strings and yield the
+ # Convert the keyword and line to UTF-8 strings and yield the
# command. Note that we take care to return regular strings in
# Python 2, to avoid other code having to differentiate between the
# str and unicode types.
keyword,ln = match.groups()
- yield (line_number, str(keyword[:-1].decode('ascii')),
- str(ln.decode('ascii')))
+ yield (line_number, to_string(keyword[:-1].decode('utf-8')),
+ to_string(ln.decode('utf-8')))
finally:
f.close()
+
def parseIntegratedTestScript(test, normalize_slashes=False,
- extra_substitutions=[]):
+ extra_substitutions=[], require_script=True):
"""parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
- information. The RUN lines also will have variable substitution performed.
+ and 'UNSUPPORTED' information. The RUN lines also will have variable
+ substitution performed. If 'require_script' is False an empty script may be
+ returned. This can be used for test formats where the actual script is
+ optional or ignored.
"""
# Get the temporary location, this is always relative to the test suite
@@ -400,6 +414,7 @@ def parseIntegratedTestScript(test, normalize_slashes=False,
# Collect the test lines from the script.
script = []
requires = []
+ unsupported = []
for line_number, command_type, ln in \
parseIntegratedTestScriptCommands(sourcepath):
if command_type == 'RUN':
@@ -424,6 +439,8 @@ def parseIntegratedTestScript(test, normalize_slashes=False,
test.xfails.extend([s.strip() for s in ln.split(',')])
elif command_type == 'REQUIRES':
requires.extend([s.strip() for s in ln.split(',')])
+ elif command_type == 'UNSUPPORTED':
+ unsupported.extend([s.strip() for s in ln.split(',')])
elif command_type == 'END':
# END commands are only honored if the rest of the line is empty.
if not ln.strip():
@@ -448,11 +465,11 @@ def parseIntegratedTestScript(test, normalize_slashes=False,
for ln in script]
# Verify the script contains a run line.
- if not script:
+ if require_script and not script:
return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
# Check for unterminated run lines.
- if script[-1][-1] == '\\':
+ if script and script[-1][-1] == '\\':
return lit.Test.Result(Test.UNRESOLVED,
"Test has unterminated run lines (with '\\')")
@@ -463,22 +480,17 @@ def parseIntegratedTestScript(test, normalize_slashes=False,
msg = ', '.join(missing_required_features)
return lit.Test.Result(Test.UNSUPPORTED,
"Test requires the following features: %s" % msg)
+ unsupported_features = [f for f in unsupported
+ if f in test.config.available_features]
+ if unsupported_features:
+ msg = ', '.join(unsupported_features)
+ return lit.Test.Result(Test.UNSUPPORTED,
+ "Test is unsupported with the following features: %s" % msg)
return script,tmpBase,execdir
-def executeShTest(test, litConfig, useExternalSh,
- extra_substitutions=[]):
- if test.config.unsupported:
- return (Test.UNSUPPORTED, 'Test is unsupported')
-
- res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
- if isinstance(res, lit.Test.Result):
- return res
- if litConfig.noExecute:
- return lit.Test.Result(Test.PASS)
-
- script, tmpBase, execdir = res
-
+def _runShTest(test, litConfig, useExternalSh,
+ script, tmpBase, execdir):
# Create the output directory if it does not already exist.
lit.util.mkdir_p(os.path.dirname(tmpBase))
@@ -506,3 +518,19 @@ def executeShTest(test, litConfig, useExternalSh,
output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
return lit.Test.Result(status, output)
+
+
+def executeShTest(test, litConfig, useExternalSh,
+ extra_substitutions=[]):
+ if test.config.unsupported:
+ return (Test.UNSUPPORTED, 'Test is unsupported')
+
+ res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
+ if isinstance(res, lit.Test.Result):
+ return res
+ if litConfig.noExecute:
+ return lit.Test.Result(Test.PASS)
+
+ script, tmpBase, execdir = res
+ return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
+
diff --git a/utils/lit/lit/TestingConfig.py b/utils/lit/lit/TestingConfig.py
index eb890674a74d..c7ef94dc11fa 100644
--- a/utils/lit/lit/TestingConfig.py
+++ b/utils/lit/lit/TestingConfig.py
@@ -17,15 +17,21 @@ class TestingConfig:
"""
# Set the environment based on the command line arguments.
environment = {
- 'LIBRARY_PATH' : os.environ.get('LIBRARY_PATH',''),
- 'LD_LIBRARY_PATH' : os.environ.get('LD_LIBRARY_PATH',''),
'PATH' : os.pathsep.join(litConfig.path +
[os.environ.get('PATH','')]),
- 'SYSTEMROOT' : os.environ.get('SYSTEMROOT',''),
- 'TERM' : os.environ.get('TERM',''),
'LLVM_DISABLE_CRASH_REPORT' : '1',
}
+ pass_vars = ['LIBRARY_PATH', 'LD_LIBRARY_PATH', 'SYSTEMROOT', 'TERM',
+ 'LD_PRELOAD', 'ASAN_OPTIONS', 'UBSAN_OPTIONS',
+ 'LSAN_OPTIONS']
+ for var in pass_vars:
+ val = os.environ.get(var, '')
+ # Check for empty string as some variables such as LD_PRELOAD cannot be empty
+ # ('') for OS's such as OpenBSD.
+ if val:
+ environment[var] = val
+
if sys.platform == 'win32':
environment.update({
'INCLUDE' : os.environ.get('INCLUDE',''),
diff --git a/utils/lit/lit/__init__.py b/utils/lit/lit/__init__.py
index 46fa82dcdf1c..c1bd76b80197 100644
--- a/utils/lit/lit/__init__.py
+++ b/utils/lit/lit/__init__.py
@@ -5,7 +5,7 @@ from .main import main
__author__ = 'Daniel Dunbar'
__email__ = 'daniel@zuster.org'
-__versioninfo__ = (0, 4, 0)
+__versioninfo__ = (0, 5, 0)
__version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev'
__all__ = []
diff --git a/utils/lit/lit/discovery.py b/utils/lit/lit/discovery.py
index 876d4f31e950..4befe582d454 100644
--- a/utils/lit/lit/discovery.py
+++ b/utils/lit/lit/discovery.py
@@ -91,7 +91,7 @@ def getLocalConfig(ts, path_in_suite, litConfig, cache):
# Otherwise, copy the current config and load the local configuration
# file into it.
- config = copy.copy(parent)
+ config = copy.deepcopy(parent)
if litConfig.debug:
litConfig.note('loading local config %r' % cfgpath)
config.load_from_path(cfgpath, litConfig)
diff --git a/utils/lit/lit/formats/googletest.py b/utils/lit/lit/formats/googletest.py
index 3d14b729ed07..59ac3c5cb370 100644
--- a/utils/lit/lit/formats/googletest.py
+++ b/utils/lit/lit/formats/googletest.py
@@ -31,7 +31,6 @@ class GoogleTest(TestFormat):
try:
lines = lit.util.capture([path, '--gtest_list_tests'],
env=localConfig.environment)
- lines = lines.decode('ascii')
if kIsWindows:
lines = lines.replace('\r', '')
lines = lines.split('\n')
diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index 6758b6037c04..f2aedc906bb1 100755
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -43,7 +43,6 @@ class TestingProgressDisplay(object):
test.getFullName())
shouldShow = test.result.code.isFailure or \
- (self.opts.show_unsupported and test.result.code.name == 'UNSUPPORTED') or \
(not self.opts.quiet and not self.opts.succinct)
if not shouldShow:
return
@@ -173,6 +172,9 @@ def main(builtinParameters = {}):
group.add_option("", "--show-unsupported", dest="show_unsupported",
help="Show unsupported tests",
action="store_true", default=False)
+ group.add_option("", "--show-xfail", dest="show_xfail",
+ help="Show tests that were expected to fail",
+ action="store_true", default=False)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
@@ -390,7 +392,12 @@ def main(builtinParameters = {}):
# Print each test in any of the failing groups.
for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
('Failing Tests', lit.Test.FAIL),
- ('Unresolved Tests', lit.Test.UNRESOLVED)):
+ ('Unresolved Tests', lit.Test.UNRESOLVED),
+ ('Unsupported Tests', lit.Test.UNSUPPORTED),
+ ('Expected Failing Tests', lit.Test.XFAIL)):
+ if (lit.Test.XFAIL == code and not opts.show_xfail) or \
+ (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
+ continue
elts = byCode.get(code)
if not elts:
continue
@@ -411,7 +418,7 @@ def main(builtinParameters = {}):
('Unsupported Tests ', lit.Test.UNSUPPORTED),
('Unresolved Tests ', lit.Test.UNRESOLVED),
('Unexpected Passes ', lit.Test.XPASS),
- ('Unexpected Failures', lit.Test.FAIL),):
+ ('Unexpected Failures', lit.Test.FAIL)):
if opts.quiet and not code.isFailure:
continue
N = len(byCode.get(code,[]))
@@ -437,7 +444,8 @@ def main(builtinParameters = {}):
xunit_output_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n")
xunit_output_file.write("<testsuites>\n")
for suite_name, suite in by_suite.items():
- xunit_output_file.write("<testsuite name='" + suite_name + "'")
+ safe_suite_name = suite_name.replace(".", "-")
+ xunit_output_file.write("<testsuite name='" + safe_suite_name + "'")
xunit_output_file.write(" tests='" + str(suite['passes'] +
suite['failures']) + "'")
xunit_output_file.write(" failures='" + str(suite['failures']) +
diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py
index 72a8b4848e08..08f7b71ae210 100644
--- a/utils/lit/lit/util.py
+++ b/utils/lit/lit/util.py
@@ -7,6 +7,21 @@ import signal
import subprocess
import sys
+def to_bytes(str):
+ # Encode to UTF-8 to get binary data.
+ return str.encode('utf-8')
+
+def to_string(bytes):
+ if isinstance(bytes, str):
+ return bytes
+ return to_bytes(bytes)
+
+def convert_string(bytes):
+ try:
+ return to_string(bytes.decode('utf-8'))
+ except UnicodeError:
+ return str(bytes)
+
def detectCPUs():
"""
Detects the number of CPUs on a system. Cribbed from pp.
@@ -51,7 +66,7 @@ def capture(args, env=None):
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=env)
out,_ = p.communicate()
- return out
+ return convert_string(out)
def which(command, paths = None):
"""which(command, [paths]) - Look up the given command in the paths string
@@ -157,14 +172,8 @@ def executeCommand(command, cwd=None, env=None):
raise KeyboardInterrupt
# Ensure the resulting output is always of string type.
- try:
- out = str(out.decode('ascii'))
- except:
- out = str(out)
- try:
- err = str(err.decode('ascii'))
- except:
- err = str(err)
+ out = convert_string(out)
+ err = convert_string(err)
return out, err, exitCode
diff --git a/utils/lit/tests/test-output.py b/utils/lit/tests/test-output.py
index adfbcd88f22a..a50442741b9d 100644
--- a/utils/lit/tests/test-output.py
+++ b/utils/lit/tests/test-output.py
@@ -1,5 +1,3 @@
-# XFAIL: python2.5
-
# RUN: %{lit} -j 1 -v %{inputs}/test-data --output %t.results.out > %t.out
# RUN: FileCheck < %t.results.out %s
diff --git a/utils/lit/tests/xunit-output.py b/utils/lit/tests/xunit-output.py
new file mode 100644
index 000000000000..87652290f47d
--- /dev/null
+++ b/utils/lit/tests/xunit-output.py
@@ -0,0 +1,10 @@
+# Check xunit output
+# RUN: %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/test-data
+# RUN: FileCheck < %t.xunit.xml %s
+
+# CHECK: <?xml version="1.0" encoding="UTF-8" ?>
+# CHECK: <testsuites>
+# CHECK: <testsuite name='test-data' tests='1' failures='0'>
+# CHECK: <testcase classname='test-data.' name='metrics.ini' time='0.00'/>
+# CHECK: </testsuite>
+# CHECK: </testsuites> \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-01 02:41:26 +0000