diff options
Diffstat (limited to 'tests/test-runner')
| -rw-r--r-- | tests/test-runner/Makefile.am | 1 | ||||
| -rw-r--r-- | tests/test-runner/bin/.gitignore | 2 | ||||
| -rw-r--r-- | tests/test-runner/bin/Makefile.am | 8 | ||||
| -rwxr-xr-x | tests/test-runner/bin/test-runner.py.in | 1056 | ||||
| -rwxr-xr-x | tests/test-runner/bin/zts-report.py.in | 390 | ||||
| -rw-r--r-- | tests/test-runner/include/Makefile.am | 5 | ||||
| -rw-r--r-- | tests/test-runner/include/logapi.shlib | 530 | ||||
| -rw-r--r-- | tests/test-runner/include/stf.shlib | 57 | ||||
| -rw-r--r-- | tests/test-runner/man/Makefile.am | 4 | ||||
| -rw-r--r-- | tests/test-runner/man/test-runner.1 | 386 |
10 files changed, 2439 insertions, 0 deletions
diff --git a/tests/test-runner/Makefile.am b/tests/test-runner/Makefile.am new file mode 100644 index 000000000000..db3d966142d6 --- /dev/null +++ b/tests/test-runner/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = bin include man diff --git a/tests/test-runner/bin/.gitignore b/tests/test-runner/bin/.gitignore new file mode 100644 index 000000000000..ff7e2f8fcc76 --- /dev/null +++ b/tests/test-runner/bin/.gitignore @@ -0,0 +1,2 @@ +test-runner.py +zts-report.py diff --git a/tests/test-runner/bin/Makefile.am b/tests/test-runner/bin/Makefile.am new file mode 100644 index 000000000000..e11e55fffdeb --- /dev/null +++ b/tests/test-runner/bin/Makefile.am @@ -0,0 +1,8 @@ +include $(top_srcdir)/config/Substfiles.am + +pkgdatadir = $(datadir)/@PACKAGE@/test-runner/bin +pkgdata_SCRIPTS = \ + test-runner.py \ + zts-report.py + +SUBSTFILES += $(pkgdata_SCRIPTS) diff --git a/tests/test-runner/bin/test-runner.py.in b/tests/test-runner/bin/test-runner.py.in new file mode 100755 index 000000000000..bbabf247c1dc --- /dev/null +++ b/tests/test-runner/bin/test-runner.py.in @@ -0,0 +1,1056 @@ +#!/usr/bin/env @PYTHON_SHEBANG@ + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2012, 2018 by Delphix. All rights reserved. +# Copyright (c) 2019 Datto Inc. +# +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# + +# some python 2.7 system don't have a configparser shim +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import os +import sys +import ctypes + +from datetime import datetime +from optparse import OptionParser +from pwd import getpwnam +from pwd import getpwuid +from select import select +from subprocess import PIPE +from subprocess import Popen +from threading import Timer +from time import time + +BASEDIR = '/var/tmp/test_results' +TESTDIR = '/usr/share/zfs/' +KILL = 'kill' +TRUE = 'true' +SUDO = 'sudo' +LOG_FILE = 'LOG_FILE' +LOG_OUT = 'LOG_OUT' +LOG_ERR = 'LOG_ERR' +LOG_FILE_OBJ = None + +# some python 2.7 system don't have a concept of monotonic time +CLOCK_MONOTONIC_RAW = 4 # see <linux/time.h> + + +class timespec(ctypes.Structure): + _fields_ = [ + ('tv_sec', ctypes.c_long), + ('tv_nsec', ctypes.c_long) + ] + + +librt = ctypes.CDLL('librt.so.1', use_errno=True) +clock_gettime = librt.clock_gettime +clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] + + +def monotonic_time(): + t = timespec() + if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0: + errno_ = ctypes.get_errno() + raise OSError(errno_, os.strerror(errno_)) + return t.tv_sec + t.tv_nsec * 1e-9 + + +class Result(object): + total = 0 + runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} + + def __init__(self): + self.starttime = None + self.returncode = None + self.runtime = '' + self.stdout = [] + self.stderr = [] + self.result = '' + + def done(self, proc, killed, reran): + """ + Finalize the results of this Cmd. + """ + Result.total += 1 + m, s = divmod(monotonic_time() - self.starttime, 60) + self.runtime = '%02d:%02d' % (m, s) + self.returncode = proc.returncode + if reran is True: + Result.runresults['RERAN'] += 1 + if killed: + self.result = 'KILLED' + Result.runresults['KILLED'] += 1 + elif self.returncode == 0: + self.result = 'PASS' + Result.runresults['PASS'] += 1 + elif self.returncode == 4: + self.result = 'SKIP' + Result.runresults['SKIP'] += 1 + elif self.returncode != 0: + self.result = 'FAIL' + Result.runresults['FAIL'] += 1 + + +class Output(object): + """ + This class is a slightly modified version of the 'Stream' class found + here: http://goo.gl/aSGfv + """ + def __init__(self, stream): + self.stream = stream + self._buf = b'' + self.lines = [] + + def fileno(self): + return self.stream.fileno() + + def read(self, drain=0): + """ + Read from the file descriptor. If 'drain' set, read until EOF. + """ + while self._read() is not None: + if not drain: + break + + def _read(self): + """ + Read up to 4k of data from this output stream. Collect the output + up to the last newline, and append it to any leftover data from a + previous call. The lines are stored as a (timestamp, data) tuple + for easy sorting/merging later. + """ + fd = self.fileno() + buf = os.read(fd, 4096) + if not buf: + return None + if b'\n' not in buf: + self._buf += buf + return [] + + buf = self._buf + buf + tmp, rest = buf.rsplit(b'\n', 1) + self._buf = rest + now = datetime.now() + rows = tmp.split(b'\n') + self.lines += [(now, r) for r in rows] + + +class Cmd(object): + verified_users = [] + + def __init__(self, pathname, identifier=None, outputdir=None, + timeout=None, user=None, tags=None): + self.pathname = pathname + self.identifier = identifier + self.outputdir = outputdir or 'BASEDIR' + """ + The timeout for tests is measured in wall-clock time + """ + self.timeout = timeout + self.user = user or '' + self.killed = False + self.reran = None + self.result = Result() + + if self.timeout is None: + self.timeout = 60 + + def __str__(self): + return '''\ +Pathname: %s +Identifier: %s +Outputdir: %s +Timeout: %d +User: %s +''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) + + def kill_cmd(self, proc, keyboard_interrupt=False): + """ + Kill a running command due to timeout, or ^C from the keyboard. If + sudo is required, this user was verified previously. + """ + self.killed = True + do_sudo = len(self.user) != 0 + signal = '-TERM' + + cmd = [SUDO, KILL, signal, str(proc.pid)] + if not do_sudo: + del cmd[0] + + try: + kp = Popen(cmd) + kp.wait() + except Exception: + pass + + """ + If this is not a user-initiated kill and the test has not been + reran before we consider if the test needs to be reran: + If the test has spent some time hibernating and didn't run the whole + length of time before being timed out we will rerun the test. + """ + if keyboard_interrupt is False and self.reran is None: + runtime = monotonic_time() - self.result.starttime + if int(self.timeout) > runtime: + self.killed = False + self.reran = False + self.run(False) + self.reran = True + + def update_cmd_privs(self, cmd, user): + """ + If a user has been specified to run this Cmd and we're not already + running as that user, prepend the appropriate sudo command to run + as that user. + """ + me = getpwuid(os.getuid()) + + if not user or user is me: + if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): + cmd += '.ksh' + if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): + cmd += '.sh' + return cmd + + if not os.path.isfile(cmd): + if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): + cmd += '.ksh' + if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): + cmd += '.sh' + + ret = '%s -E -u %s %s' % (SUDO, user, cmd) + return ret.split(' ') + + def collect_output(self, proc): + """ + Read from stdout/stderr as data becomes available, until the + process is no longer running. Return the lines from the stdout and + stderr Output objects. + """ + out = Output(proc.stdout) + err = Output(proc.stderr) + res = [] + while proc.returncode is None: + proc.poll() + res = select([out, err], [], [], .1) + for fd in res[0]: + fd.read() + for fd in res[0]: + fd.read(drain=1) + + return out.lines, err.lines + + def run(self, dryrun): + """ + This is the main function that runs each individual test. + Determine whether or not the command requires sudo, and modify it + if needed. Run the command, and update the result object. + """ + if dryrun is True: + print(self) + return + + privcmd = self.update_cmd_privs(self.pathname, self.user) + try: + old = os.umask(0) + if not os.path.isdir(self.outputdir): + os.makedirs(self.outputdir, mode=0o777) + os.umask(old) + except OSError as e: + fail('%s' % e) + + self.result.starttime = monotonic_time() + proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) + # Allow a special timeout value of 0 to mean infinity + if int(self.timeout) == 0: + self.timeout = sys.maxsize + t = Timer(int(self.timeout), self.kill_cmd, [proc]) + + try: + t.start() + self.result.stdout, self.result.stderr = self.collect_output(proc) + except KeyboardInterrupt: + self.kill_cmd(proc, True) + fail('\nRun terminated at user request.') + finally: + t.cancel() + + if self.reran is not False: + self.result.done(proc, self.killed, self.reran) + + def skip(self): + """ + Initialize enough of the test result that we can log a skipped + command. + """ + Result.total += 1 + Result.runresults['SKIP'] += 1 + self.result.stdout = self.result.stderr = [] + self.result.starttime = monotonic_time() + m, s = divmod(monotonic_time() - self.result.starttime, 60) + self.result.runtime = '%02d:%02d' % (m, s) + self.result.result = 'SKIP' + + def log(self, options, suppress_console=False): + """ + This function is responsible for writing all output. This includes + the console output, the logfile of all results (with timestamped + merged stdout and stderr), and for each test, the unmodified + stdout/stderr/merged in its own file. + """ + + logname = getpwuid(os.getuid()).pw_name + rer = '' + if self.reran is True: + rer = ' (RERAN)' + user = ' (run as %s)' % (self.user if len(self.user) else logname) + if self.identifier: + msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) + else: + msga = 'Test: %s%s ' % (self.pathname, user) + msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) + pad = ' ' * (80 - (len(msga) + len(msgb))) + result_line = msga + pad + msgb + + # The result line is always written to the log file. If -q was + # specified only failures are written to the console, otherwise + # the result line is written to the console. The console output + # may be suppressed by calling log() with suppress_console=True. + write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) + if not suppress_console: + if not options.quiet: + write_log(result_line, LOG_OUT) + elif options.quiet and self.result.result != 'PASS': + write_log(result_line, LOG_OUT) + + lines = sorted(self.result.stdout + self.result.stderr, + key=lambda x: x[0]) + + # Write timestamped output (stdout and stderr) to the logfile + for dt, line in lines: + timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], + encoding='utf-8') + write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) + + # Write the separate stdout/stderr/merged files, if the data exists + if len(self.result.stdout): + with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: + for _, line in self.result.stdout: + os.write(out.fileno(), b'%s\n' % line) + if len(self.result.stderr): + with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: + for _, line in self.result.stderr: + os.write(err.fileno(), b'%s\n' % line) + if len(self.result.stdout) and len(self.result.stderr): + with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: + for _, line in lines: + os.write(merged.fileno(), b'%s\n' % line) + + +class Test(Cmd): + props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', + 'post_user', 'failsafe', 'failsafe_user', 'tags'] + + def __init__(self, pathname, + pre=None, pre_user=None, post=None, post_user=None, + failsafe=None, failsafe_user=None, tags=None, **kwargs): + super(Test, self).__init__(pathname, **kwargs) + self.pre = pre or '' + self.pre_user = pre_user or '' + self.post = post or '' + self.post_user = post_user or '' + self.failsafe = failsafe or '' + self.failsafe_user = failsafe_user or '' + self.tags = tags or [] + + def __str__(self): + post_user = pre_user = failsafe_user = '' + if len(self.pre_user): + pre_user = ' (as %s)' % (self.pre_user) + if len(self.post_user): + post_user = ' (as %s)' % (self.post_user) + if len(self.failsafe_user): + failsafe_user = ' (as %s)' % (self.failsafe_user) + return '''\ +Pathname: %s +Identifier: %s +Outputdir: %s +Timeout: %d +User: %s +Pre: %s%s +Post: %s%s +Failsafe: %s%s +Tags: %s +''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, + self.pre, pre_user, self.post, post_user, self.failsafe, + failsafe_user, self.tags) + + def verify(self): + """ + Check the pre/post/failsafe scripts, user and Test. Omit the Test from + this run if there are any problems. + """ + files = [self.pre, self.pathname, self.post, self.failsafe] + users = [self.pre_user, self.user, self.post_user, self.failsafe_user] + + for f in [f for f in files if len(f)]: + if not verify_file(f): + write_log("Warning: Test '%s' not added to this run because" + " it failed verification.\n" % f, LOG_ERR) + return False + + for user in [user for user in users if len(user)]: + if not verify_user(user): + write_log("Not adding Test '%s' to this run.\n" % + self.pathname, LOG_ERR) + return False + + return True + + def run(self, options): + """ + Create Cmd instances for the pre/post/failsafe scripts. If the pre + script doesn't pass, skip this Test. Run the post script regardless. + If the Test is killed, also run the failsafe script. + """ + odir = os.path.join(self.outputdir, os.path.basename(self.pre)) + pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, + timeout=self.timeout, user=self.pre_user) + test = Cmd(self.pathname, identifier=self.identifier, + outputdir=self.outputdir, timeout=self.timeout, + user=self.user) + odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) + failsafe = Cmd(self.failsafe, identifier=self.identifier, + outputdir=odir, timeout=self.timeout, + user=self.failsafe_user) + odir = os.path.join(self.outputdir, os.path.basename(self.post)) + posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, + timeout=self.timeout, user=self.post_user) + + cont = True + if len(pretest.pathname): + pretest.run(options.dryrun) + cont = pretest.result.result == 'PASS' + pretest.log(options) + + if cont: + test.run(options.dryrun) + if test.result.result == 'KILLED' and len(failsafe.pathname): + failsafe.run(options.dryrun) + failsafe.log(options, suppress_console=True) + else: + test.skip() + + test.log(options) + + if len(posttest.pathname): + posttest.run(options.dryrun) + posttest.log(options) + + +class TestGroup(Test): + props = Test.props + ['tests'] + + def __init__(self, pathname, tests=None, **kwargs): + super(TestGroup, self).__init__(pathname, **kwargs) + self.tests = tests or [] + + def __str__(self): + post_user = pre_user = failsafe_user = '' + if len(self.pre_user): + pre_user = ' (as %s)' % (self.pre_user) + if len(self.post_user): + post_user = ' (as %s)' % (self.post_user) + if len(self.failsafe_user): + failsafe_user = ' (as %s)' % (self.failsafe_user) + return '''\ +Pathname: %s +Identifier: %s +Outputdir: %s +Tests: %s +Timeout: %s +User: %s +Pre: %s%s +Post: %s%s +Failsafe: %s%s +Tags: %s +''' % (self.pathname, self.identifier, self.outputdir, self.tests, + self.timeout, self.user, self.pre, pre_user, self.post, post_user, + self.failsafe, failsafe_user, self.tags) + + def verify(self): + """ + Check the pre/post/failsafe scripts, user and tests in this TestGroup. + Omit the TestGroup entirely, or simply delete the relevant tests in the + group, if that's all that's required. + """ + # If the pre/post/failsafe scripts are relative pathnames, convert to + # absolute, so they stand a chance of passing verification. + if len(self.pre) and not os.path.isabs(self.pre): + self.pre = os.path.join(self.pathname, self.pre) + if len(self.post) and not os.path.isabs(self.post): + self.post = os.path.join(self.pathname, self.post) + if len(self.failsafe) and not os.path.isabs(self.failsafe): + self.post = os.path.join(self.pathname, self.post) + + auxfiles = [self.pre, self.post, self.failsafe] + users = [self.pre_user, self.user, self.post_user, self.failsafe_user] + + for f in [f for f in auxfiles if len(f)]: + if f != self.failsafe and self.pathname != os.path.dirname(f): + write_log("Warning: TestGroup '%s' not added to this run. " + "Auxiliary script '%s' exists in a different " + "directory.\n" % (self.pathname, f), LOG_ERR) + return False + + if not verify_file(f): + write_log("Warning: TestGroup '%s' not added to this run. " + "Auxiliary script '%s' failed verification.\n" % + (self.pathname, f), LOG_ERR) + return False + + for user in [user for user in users if len(user)]: + if not verify_user(user): + write_log("Not adding TestGroup '%s' to this run.\n" % + self.pathname, LOG_ERR) + return False + + # If one of the tests is invalid, delete it, log it, and drive on. + for test in self.tests: + if not verify_file(os.path.join(self.pathname, test)): + del self.tests[self.tests.index(test)] + write_log("Warning: Test '%s' removed from TestGroup '%s' " + "because it failed verification.\n" % + (test, self.pathname), LOG_ERR) + + return len(self.tests) != 0 + + def run(self, options): + """ + Create Cmd instances for the pre/post/failsafe scripts. If the pre + script doesn't pass, skip all the tests in this TestGroup. Run the + post script regardless. Run the failsafe script when a test is killed. + """ + # tags assigned to this test group also include the test names + if options.tags and not set(self.tags).intersection(set(options.tags)): + return + + odir = os.path.join(self.outputdir, os.path.basename(self.pre)) + pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, + user=self.pre_user, identifier=self.identifier) + odir = os.path.join(self.outputdir, os.path.basename(self.post)) + posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, + user=self.post_user, identifier=self.identifier) + + cont = True + if len(pretest.pathname): + pretest.run(options.dryrun) + cont = pretest.result.result == 'PASS' + pretest.log(options) + + for fname in self.tests: + odir = os.path.join(self.outputdir, fname) + test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, + timeout=self.timeout, user=self.user, + identifier=self.identifier) + odir = os.path.join(odir, os.path.basename(self.failsafe)) + failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, + user=self.failsafe_user, identifier=self.identifier) + if cont: + test.run(options.dryrun) + if test.result.result == 'KILLED' and len(failsafe.pathname): + failsafe.run(options.dryrun) + failsafe.log(options, suppress_console=True) + else: + test.skip() + + test.log(options) + + if len(posttest.pathname): + posttest.run(options.dryrun) + posttest.log(options) + + +class TestRun(object): + props = ['quiet', 'outputdir'] + + def __init__(self, options): + self.tests = {} + self.testgroups = {} + self.starttime = time() + self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') + self.outputdir = os.path.join(options.outputdir, self.timestamp) + self.setup_logging(options) + self.defaults = [ + ('outputdir', BASEDIR), + ('quiet', False), + ('timeout', 60), + ('user', ''), + ('pre', ''), + ('pre_user', ''), + ('post', ''), + ('post_user', ''), + ('failsafe', ''), + ('failsafe_user', ''), + ('tags', []) + ] + + def __str__(self): + s = 'TestRun:\n outputdir: %s\n' % self.outputdir + s += 'TESTS:\n' + for key in sorted(self.tests.keys()): + s += '%s%s' % (self.tests[key].__str__(), '\n') + s += 'TESTGROUPS:\n' + for key in sorted(self.testgroups.keys()): + s += '%s%s' % (self.testgroups[key].__str__(), '\n') + return s + + def addtest(self, pathname, options): + """ + Create a new Test, and apply any properties that were passed in + from the command line. If it passes verification, add it to the + TestRun. + """ + test = Test(pathname) + for prop in Test.props: + setattr(test, prop, getattr(options, prop)) + + if test.verify(): + self.tests[pathname] = test + + def addtestgroup(self, dirname, filenames, options): + """ + Create a new TestGroup, and apply any properties that were passed + in from the command line. If it passes verification, add it to the + TestRun. + """ + if dirname not in self.testgroups: + testgroup = TestGroup(dirname) + for prop in Test.props: + setattr(testgroup, prop, getattr(options, prop)) + + # Prevent pre/post/failsafe scripts from running as regular tests + for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: + if f in filenames: + del filenames[filenames.index(f)] + + self.testgroups[dirname] = testgroup + self.testgroups[dirname].tests = sorted(filenames) + + testgroup.verify() + + def read(self, options): + """ + Read in the specified runfiles, and apply the TestRun properties + listed in the 'DEFAULT' section to our TestRun. Then read each + section, and apply the appropriate properties to the Test or + TestGroup. Properties from individual sections override those set + in the 'DEFAULT' section. If the Test or TestGroup passes + verification, add it to the TestRun. + """ + config = configparser.RawConfigParser() + parsed = config.read(options.runfiles) + failed = options.runfiles - set(parsed) + if len(failed): + files = ' '.join(sorted(failed)) + fail("Couldn't read config files: %s" % files) + + for opt in TestRun.props: + if config.has_option('DEFAULT', opt): + setattr(self, opt, config.get('DEFAULT', opt)) + self.outputdir = os.path.join(self.outputdir, self.timestamp) + + testdir = options.testdir + + for section in config.sections(): + if 'tests' in config.options(section): + parts = section.split(':', 1) + sectiondir = parts[0] + identifier = parts[1] if len(parts) == 2 else None + if os.path.isdir(sectiondir): + pathname = sectiondir + elif os.path.isdir(os.path.join(testdir, sectiondir)): + pathname = os.path.join(testdir, sectiondir) + else: + pathname = sectiondir + + testgroup = TestGroup(os.path.abspath(pathname), + identifier=identifier) + for prop in TestGroup.props: + for sect in ['DEFAULT', section]: + if config.has_option(sect, prop): + if prop == 'tags': + setattr(testgroup, prop, + eval(config.get(sect, prop))) + elif prop == 'failsafe': + failsafe = config.get(sect, prop) + setattr(testgroup, prop, + os.path.join(testdir, failsafe)) + else: + setattr(testgroup, prop, + config.get(sect, prop)) + + # Repopulate tests using eval to convert the string to a list + testgroup.tests = eval(config.get(section, 'tests')) + + if testgroup.verify(): + self.testgroups[section] = testgroup + else: + test = Test(section) + for prop in Test.props: + for sect in ['DEFAULT', section]: + if config.has_option(sect, prop): + if prop == 'failsafe': + failsafe = config.get(sect, prop) + setattr(test, prop, + os.path.join(testdir, failsafe)) + else: + setattr(test, prop, config.get(sect, prop)) + + if test.verify(): + self.tests[section] = test + + def write(self, options): + """ + Create a configuration file for editing and later use. The + 'DEFAULT' section of the config file is created from the + properties that were specified on the command line. Tests are + simply added as sections that inherit everything from the + 'DEFAULT' section. TestGroups are the same, except they get an + option including all the tests to run in that directory. + """ + + defaults = dict([(prop, getattr(options, prop)) for prop, _ in + self.defaults]) + config = configparser.RawConfigParser(defaults) + + for test in sorted(self.tests.keys()): + config.add_section(test) + + for testgroup in sorted(self.testgroups.keys()): + config.add_section(testgroup) + config.set(testgroup, 'tests', self.testgroups[testgroup].tests) + + try: + with open(options.template, 'w') as f: + return config.write(f) + except IOError: + fail('Could not open \'%s\' for writing.' % options.template) + + def complete_outputdirs(self): + """ + Collect all the pathnames for Tests, and TestGroups. Work + backwards one pathname component at a time, to create a unique + directory name in which to deposit test output. Tests will be able + to write output files directly in the newly modified outputdir. + TestGroups will be able to create one subdirectory per test in the + outputdir, and are guaranteed uniqueness because a group can only + contain files in one directory. Pre and post tests will create a + directory rooted at the outputdir of the Test or TestGroup in + question for their output. Failsafe scripts will create a directory + rooted at the outputdir of each Test for their output. + """ + done = False + components = 0 + tmp_dict = dict(list(self.tests.items()) + + list(self.testgroups.items())) + total = len(tmp_dict) + base = self.outputdir + + while not done: + paths = [] + components -= 1 + for testfile in list(tmp_dict.keys()): + uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') + if uniq not in paths: + paths.append(uniq) + tmp_dict[testfile].outputdir = os.path.join(base, uniq) + else: + break + done = total == len(paths) + + def setup_logging(self, options): + """ + This function creates the output directory and gets a file object + for the logfile. This function must be called before write_log() + can be used. + """ + if options.dryrun is True: + return + + global LOG_FILE_OBJ + if options.cmd != 'wrconfig': + try: + old = os.umask(0) + os.makedirs(self.outputdir, mode=0o777) + os.umask(old) + filename = os.path.join(self.outputdir, 'log') + LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') + except OSError as e: + fail('%s' % e) + + def run(self, options): + """ + Walk through all the Tests and TestGroups, calling run(). + """ + try: + os.chdir(self.outputdir) + except OSError: + fail('Could not change to directory %s' % self.outputdir) + # make a symlink to the output for the currently running test + logsymlink = os.path.join(self.outputdir, '../current') + if os.path.islink(logsymlink): + os.unlink(logsymlink) + if not os.path.exists(logsymlink): + os.symlink(self.outputdir, logsymlink) + else: + write_log('Could not make a symlink to directory %s\n' % + self.outputdir, LOG_ERR) + iteration = 0 + while iteration < options.iterations: + for test in sorted(self.tests.keys()): + self.tests[test].run(options) + for testgroup in sorted(self.testgroups.keys()): + self.testgroups[testgroup].run(options) + iteration += 1 + + def summary(self): + if Result.total == 0: + return 2 + + print('\nResults Summary') + for key in list(Result.runresults.keys()): + if Result.runresults[key] != 0: + print('%s\t% 4d' % (key, Result.runresults[key])) + + m, s = divmod(time() - self.starttime, 60) + h, m = divmod(m, 60) + print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) + print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / + float(Result.total)) * 100)) + print('Log directory:\t%s' % self.outputdir) + + if Result.runresults['FAIL'] > 0: + return 1 + + if Result.runresults['KILLED'] > 0: + return 1 + + if Result.runresults['RERAN'] > 0: + return 3 + + return 0 + + +def write_log(msg, target): + """ + Write the provided message to standard out, standard error or + the logfile. If specifying LOG_FILE, then `msg` must be a bytes + like object. This way we can still handle output from tests that + may be in unexpected encodings. + """ + if target == LOG_OUT: + os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) + elif target == LOG_ERR: + os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) + elif target == LOG_FILE: + os.write(LOG_FILE_OBJ.fileno(), msg) + else: + fail('log_msg called with unknown target "%s"' % target) + + +def verify_file(pathname): + """ + Verify that the supplied pathname is an executable regular file. + """ + if os.path.isdir(pathname) or os.path.islink(pathname): + return False + + for ext in '', '.ksh', '.sh': + script_path = pathname + ext + if os.path.isfile(script_path) and os.access(script_path, os.X_OK): + return True + + return False + + +def verify_user(user): + """ + Verify that the specified user exists on this system, and can execute + sudo without being prompted for a password. + """ + testcmd = [SUDO, '-n', '-u', user, TRUE] + + if user in Cmd.verified_users: + return True + + try: + getpwnam(user) + except KeyError: + write_log("Warning: user '%s' does not exist.\n" % user, + LOG_ERR) + return False + + p = Popen(testcmd) + p.wait() + if p.returncode != 0: + write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, + LOG_ERR) + return False + else: + Cmd.verified_users.append(user) + + return True + + +def find_tests(testrun, options): + """ + For the given list of pathnames, add files as Tests. For directories, + if do_groups is True, add the directory as a TestGroup. If False, + recursively search for executable files. + """ + + for p in sorted(options.pathnames): + if os.path.isdir(p): + for dirname, _, filenames in os.walk(p): + if options.do_groups: + testrun.addtestgroup(dirname, filenames, options) + else: + for f in sorted(filenames): + testrun.addtest(os.path.join(dirname, f), options) + else: + testrun.addtest(p, options) + + +def fail(retstr, ret=1): + print('%s: %s' % (sys.argv[0], retstr)) + exit(ret) + + +def options_cb(option, opt_str, value, parser): + path_options = ['outputdir', 'template', 'testdir'] + + if option.dest == 'runfiles' and '-w' in parser.rargs or \ + option.dest == 'template' and '-c' in parser.rargs: + fail('-c and -w are mutually exclusive.') + + if opt_str in parser.rargs: + fail('%s may only be specified once.' % opt_str) + + if option.dest == 'runfiles': + parser.values.cmd = 'rdconfig' + value = set(os.path.abspath(p) for p in value.split(',')) + if option.dest == 'template': + parser.values.cmd = 'wrconfig' + if option.dest == 'tags': + value = [x.strip() for x in value.split(',')] + + if option.dest in path_options: + setattr(parser.values, option.dest, os.path.abspath(value)) + else: + setattr(parser.values, option.dest, value) + + +def parse_args(): + parser = OptionParser() + parser.add_option('-c', action='callback', callback=options_cb, + type='string', dest='runfiles', metavar='runfiles', + help='Specify tests to run via config files.') + parser.add_option('-d', action='store_true', default=False, dest='dryrun', + help='Dry run. Print tests, but take no other action.') + parser.add_option('-g', action='store_true', default=False, + dest='do_groups', help='Make directories TestGroups.') + parser.add_option('-o', action='callback', callback=options_cb, + default=BASEDIR, dest='outputdir', type='string', + metavar='outputdir', help='Specify an output directory.') + parser.add_option('-i', action='callback', callback=options_cb, + default=TESTDIR, dest='testdir', type='string', + metavar='testdir', help='Specify a test directory.') + parser.add_option('-p', action='callback', callback=options_cb, + default='', dest='pre', metavar='script', + type='string', help='Specify a pre script.') + parser.add_option('-P', action='callback', callback=options_cb, + default='', dest='post', metavar='script', + type='string', help='Specify a post script.') + parser.add_option('-q', action='store_true', default=False, dest='quiet', + help='Silence on the console during a test run.') + parser.add_option('-s', action='callback', callback=options_cb, + default='', dest='failsafe', metavar='script', + type='string', help='Specify a failsafe script.') + parser.add_option('-S', action='callback', callback=options_cb, + default='', dest='failsafe_user', + metavar='failsafe_user', type='string', + help='Specify a user to execute the failsafe script.') + parser.add_option('-t', action='callback', callback=options_cb, default=60, + dest='timeout', metavar='seconds', type='int', + help='Timeout (in seconds) for an individual test.') + parser.add_option('-u', action='callback', callback=options_cb, + default='', dest='user', metavar='user', type='string', + help='Specify a different user name to run as.') + parser.add_option('-w', action='callback', callback=options_cb, + default=None, dest='template', metavar='template', + type='string', help='Create a new config file.') + parser.add_option('-x', action='callback', callback=options_cb, default='', + dest='pre_user', metavar='pre_user', type='string', + help='Specify a user to execute the pre script.') + parser.add_option('-X', action='callback', callback=options_cb, default='', + dest='post_user', metavar='post_user', type='string', + help='Specify a user to execute the post script.') + parser.add_option('-T', action='callback', callback=options_cb, default='', + dest='tags', metavar='tags', type='string', + help='Specify tags to execute specific test groups.') + parser.add_option('-I', action='callback', callback=options_cb, default=1, + dest='iterations', metavar='iterations', type='int', + help='Number of times to run the test run.') + (options, pathnames) = parser.parse_args() + + if not options.runfiles and not options.template: + options.cmd = 'runtests' + + if options.runfiles and len(pathnames): + fail('Extraneous arguments.') + + options.pathnames = [os.path.abspath(path) for path in pathnames] + + return options + + +def main(): + options = parse_args() + testrun = TestRun(options) + + if options.cmd == 'runtests': + find_tests(testrun, options) + elif options.cmd == 'rdconfig': + testrun.read(options) + elif options.cmd == 'wrconfig': + find_tests(testrun, options) + testrun.write(options) + exit(0) + else: + fail('Unknown command specified') + + testrun.complete_outputdirs() + testrun.run(options) + exit(testrun.summary()) + + +if __name__ == '__main__': + main() diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in new file mode 100755 index 000000000000..6b5cd191c5e0 --- /dev/null +++ b/tests/test-runner/bin/zts-report.py.in @@ -0,0 +1,390 @@ +#!/usr/bin/env @PYTHON_SHEBANG@ + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Delphix. All rights reserved. +# Copyright (c) 2018 by Lawrence Livermore National Security, LLC. +# +# This script must remain compatible with Python 2.6+ and Python 3.4+. +# + +import os +import re +import sys + +# +# This script parses the stdout of zfstest, which has this format: +# +# Test: /path/to/testa (run as root) [00:00] [PASS] +# Test: /path/to/testb (run as jkennedy) [00:00] [PASS] +# Test: /path/to/testc (run as root) [00:00] [FAIL] +# [...many more results...] +# +# Results Summary +# FAIL 22 +# SKIP 32 +# PASS 1156 +# +# Running Time: 02:50:31 +# Percent passed: 95.5% +# Log directory: /var/tmp/test_results/20180615T205926 +# + +# +# Common generic reasons for a test or test group to be skipped. +# +# Some test cases are known to fail in ways which are not harmful or dangerous. +# In these cases simply mark the test as a known failure until it can be +# updated and the issue resolved. Note that it's preferable to open a unique +# issue on the GitHub issue tracker for each test case failure. +# +known_reason = 'Known issue' + +# +# Some tests require that a test user be able to execute the zfs utilities. +# This may not be possible when testing in-tree due to the default permissions +# on the user's home directory. When testing this can be resolved by granting +# group read access. +# +# chmod 0750 $HOME +# +exec_reason = 'Test user execute permissions required for utilities' + +# +# Some tests require a minimum python version of 3.5 and will be skipped when +# the default system version is too old. There may also be tests which require +# additional python modules be installed, for example python-cffi is required +# by the pyzfs tests. +# +python_reason = 'Python v3.5 or newer required' +python_deps_reason = 'Python modules missing: python-cffi' + +# +# Some tests require the O_TMPFILE flag which was first introduced in the +# 3.11 kernel. +# +tmpfile_reason = 'Kernel O_TMPFILE support required' + +# +# Some tests require that the NFS client and server utilities be installed. +# +share_reason = 'NFS client and server utilities required' + +# +# Some tests require that the lsattr utility support the project id feature. +# +project_id_reason = 'lsattr with set/show project ID required' + +# +# Some tests require that the kernel support user namespaces. +# +user_ns_reason = 'Kernel user namespace support required' + +# +# Some rewind tests can fail since nothing guarantees that old MOS blocks +# are not overwritten. Snapshots protect datasets and data files but not +# the MOS. Reasonable efforts are made in the test case to increase the +# odds that some txgs will have their MOS data left untouched, but it is +# never a sure thing. +# +rewind_reason = 'Arbitrary pool rewind is not guaranteed' + +# +# Some tests may by structured in a way that relies on exact knowledge +# of how much free space in available in a pool. These tests cannot be +# made completely reliable because the internal details of how free space +# is managed are not exposed to user space. +# +enospc_reason = 'Exact free space reporting is not guaranteed' + +# +# Some tests require a minimum version of the fio benchmark utility. +# Older distributions such as CentOS 6.x only provide fio-2.0.13. +# +fio_reason = 'Fio v2.3 or newer required' + +# +# Some tests require that the DISKS provided support the discard operation. +# Normally this is not an issue because loop back devices are used for DISKS +# and they support discard (TRIM/UNMAP). +# +trim_reason = 'DISKS must support discard (TRIM/UNMAP)' + +# +# Some tests are not applicable to a platform or need to be updated to operate +# in the manor required by the platform. Any tests which are skipped for this +# reason will be suppressed in the final analysis output. +# +na_reason = "Not applicable" + +summary = { + 'total': float(0), + 'passed': float(0), + 'logfile': "Could not determine logfile location." +} + +# +# These tests are known to fail, thus we use this list to prevent these +# failures from failing the job as a whole; only unexpected failures +# bubble up to cause this script to exit with a non-zero exit status. +# +# Format: { 'test-name': ['expected result', 'issue-number | reason'] } +# +# For each known failure it is recommended to link to a GitHub issue by +# setting the reason to the issue number. Alternately, one of the generic +# reasons listed above can be used. +# +known = { + 'casenorm/mixed_none_lookup_ci': ['FAIL', '7633'], + 'casenorm/mixed_formd_lookup_ci': ['FAIL', '7633'], + 'cli_root/zfs_unshare/zfs_unshare_002_pos': ['SKIP', na_reason], + 'cli_root/zfs_unshare/zfs_unshare_006_pos': ['SKIP', na_reason], + 'cli_user/misc/zfs_share_001_neg': ['SKIP', na_reason], + 'cli_user/misc/zfs_unshare_001_neg': ['SKIP', na_reason], + 'privilege/setup': ['SKIP', na_reason], + 'refreserv/refreserv_004_pos': ['FAIL', known_reason], + 'rootpool/setup': ['SKIP', na_reason], + 'rsend/rsend_008_pos': ['SKIP', '6066'], + 'vdev_zaps/vdev_zaps_007_pos': ['FAIL', known_reason], +} + +if sys.platform.startswith('freebsd'): + known.update({ + 'cli_root/zpool_wait/zpool_wait_trim_basic': ['SKIP', trim_reason], + 'cli_root/zpool_wait/zpool_wait_trim_cancel': ['SKIP', trim_reason], + 'cli_root/zpool_wait/zpool_wait_trim_flag': ['SKIP', trim_reason], + 'link_count/link_count_001': ['SKIP', na_reason], + }) +elif sys.platform.startswith('linux'): + known.update({ + 'casenorm/mixed_formd_lookup': ['FAIL', '7633'], + 'casenorm/mixed_formd_delete': ['FAIL', '7633'], + 'casenorm/sensitive_formd_lookup': ['FAIL', '7633'], + 'casenorm/sensitive_formd_delete': ['FAIL', '7633'], + 'removal/removal_with_zdb': ['SKIP', known_reason], + }) + + +# +# These tests may occasionally fail or be skipped. We want there failures +# to be reported but only unexpected failures should bubble up to cause +# this script to exit with a non-zero exit status. +# +# Format: { 'test-name': ['expected result', 'issue-number | reason'] } +# +# For each known failure it is recommended to link to a GitHub issue by +# setting the reason to the issue number. Alternately, one of the generic +# reasons listed above can be used. +# +maybe = { + 'alloc_class/alloc_class_012_pos': ['FAIL', '9142'], + 'alloc_class/alloc_class_013_pos': ['FAIL', '9142'], + 'chattr/setup': ['SKIP', exec_reason], + 'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason], + 'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason], + 'cli_root/zfs_get/zfs_get_009_pos': ['SKIP', '5479'], + 'cli_root/zfs_share/setup': ['SKIP', share_reason], + 'cli_root/zfs_snapshot/zfs_snapshot_002_neg': ['FAIL', known_reason], + 'cli_root/zfs_unshare/setup': ['SKIP', share_reason], + 'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason], + 'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'], + 'cli_root/zpool_import/import_rewind_device_replaced': + ['FAIL', rewind_reason], + 'cli_root/zpool_import/import_rewind_config_changed': + ['FAIL', rewind_reason], + 'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'], + 'cli_root/zpool_trim/setup': ['SKIP', trim_reason], + 'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', '6141'], + 'delegate/setup': ['SKIP', exec_reason], + 'history/history_004_pos': ['FAIL', '7026'], + 'history/history_005_neg': ['FAIL', '6680'], + 'history/history_006_neg': ['FAIL', '5657'], + 'history/history_008_pos': ['FAIL', known_reason], + 'history/history_010_pos': ['SKIP', exec_reason], + 'io/mmap': ['SKIP', fio_reason], + 'largest_pool/largest_pool_001_pos': ['FAIL', known_reason], + 'mmp/mmp_on_uberblocks': ['FAIL', known_reason], + 'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason], + 'no_space/enospc_002_pos': ['FAIL', enospc_reason], + 'projectquota/setup': ['SKIP', exec_reason], + 'redundancy/redundancy_004_neg': ['FAIL', '7290'], + 'reservation/reservation_008_pos': ['FAIL', '7741'], + 'reservation/reservation_018_pos': ['FAIL', '5642'], + 'rsend/rsend_019_pos': ['FAIL', '6086'], + 'rsend/rsend_020_pos': ['FAIL', '6446'], + 'rsend/rsend_021_pos': ['FAIL', '6446'], + 'rsend/rsend_024_pos': ['FAIL', '5665'], + 'rsend/send-c_volume': ['FAIL', '6087'], + 'rsend/send_partial_dataset': ['FAIL', known_reason], + 'snapshot/clone_001_pos': ['FAIL', known_reason], + 'snapshot/snapshot_009_pos': ['FAIL', '7961'], + 'snapshot/snapshot_010_pos': ['FAIL', '7961'], + 'snapused/snapused_004_pos': ['FAIL', '5513'], + 'tmpfile/setup': ['SKIP', tmpfile_reason], + 'threadsappend/threadsappend_001_pos': ['FAIL', '6136'], + 'trim/setup': ['SKIP', trim_reason], + 'upgrade/upgrade_projectquota_001_pos': ['SKIP', project_id_reason], + 'user_namespace/setup': ['SKIP', user_ns_reason], + 'userquota/setup': ['SKIP', exec_reason], + 'vdev_zaps/vdev_zaps_004_pos': ['FAIL', '6935'], + 'zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos': ['FAIL', '5848'], + 'pam/setup': ['SKIP', "pamtester might be not available"], +} + +if sys.platform.startswith('freebsd'): + maybe.update({ + 'cli_root/zfs_copies/zfs_copies_002_pos': ['FAIL', known_reason], + 'cli_root/zfs_inherit/zfs_inherit_001_neg': ['FAIL', known_reason], + 'cli_root/zfs_share/zfs_share_011_pos': ['FAIL', known_reason], + 'cli_root/zfs_share/zfs_share_concurrent_shares': + ['FAIL', known_reason], + 'cli_root/zpool_import/zpool_import_012_pos': ['FAIL', known_reason], + 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], + 'removal/removal_condense_export': ['FAIL', known_reason], + 'removal/removal_with_export': ['FAIL', known_reason], + 'resilver/resilver_restart_001': ['FAIL', known_reason], + 'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason], + }) +elif sys.platform.startswith('linux'): + maybe.update({ + 'alloc_class/alloc_class_009_pos': ['FAIL', known_reason], + 'alloc_class/alloc_class_010_pos': ['FAIL', known_reason], + 'alloc_class/alloc_class_011_neg': ['FAIL', known_reason], + 'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason], + 'cli_root/zpool_expand/zpool_expand_001_pos': ['FAIL', known_reason], + 'cli_root/zpool_expand/zpool_expand_005_pos': ['FAIL', known_reason], + 'cli_root/zpool_reopen/zpool_reopen_003_pos': ['FAIL', known_reason], + 'limits/filesystem_limit': ['SKIP', known_reason], + 'limits/snapshot_limit': ['SKIP', known_reason], + 'mmp/mmp_exported_import': ['FAIL', known_reason], + 'mmp/mmp_inactive_import': ['FAIL', known_reason], + 'refreserv/refreserv_raidz': ['FAIL', known_reason], + 'rsend/rsend_007_pos': ['FAIL', known_reason], + 'rsend/rsend_010_pos': ['FAIL', known_reason], + 'rsend/rsend_011_pos': ['FAIL', known_reason], + 'snapshot/rollback_003_pos': ['FAIL', known_reason], + }) + + +def usage(s): + print(s) + sys.exit(1) + + +def process_results(pathname): + try: + f = open(pathname) + except IOError as e: + print('Error opening file: %s' % e) + sys.exit(1) + + prefix = '/zfs-tests/tests/functional/' + pattern = \ + r'^Test(?:\s+\(\S+\))?:' + \ + r'\s*\S*%s(\S+)\s*\(run as (\S+)\)\s*\[(\S+)\]\s*\[(\S+)\]' \ + % prefix + pattern_log = r'^\s*Log directory:\s*(\S*)' + + d = {} + for line in f.readlines(): + m = re.match(pattern, line) + if m and len(m.groups()) == 4: + summary['total'] += 1 + if m.group(4) == "PASS": + summary['passed'] += 1 + d[m.group(1)] = m.group(4) + continue + + m = re.match(pattern_log, line) + if m: + summary['logfile'] = m.group(1) + + return d + + +if __name__ == "__main__": + if len(sys.argv) != 2: + usage('usage: %s <pathname>' % sys.argv[0]) + results = process_results(sys.argv[1]) + + if summary['total'] == 0: + print("\n\nNo test results were found.") + print("Log directory: %s" % summary['logfile']) + sys.exit(0) + + expected = [] + unexpected = [] + + for test in list(results.keys()): + if results[test] == "PASS": + continue + + setup = test.replace(os.path.basename(test), "setup") + if results[test] == "SKIP" and test != setup: + if setup in known and known[setup][0] == "SKIP": + continue + if setup in maybe and maybe[setup][0] == "SKIP": + continue + + if ((test not in known or results[test] not in known[test][0]) and + (test not in maybe or results[test] not in maybe[test][0])): + unexpected.append(test) + else: + expected.append(test) + + print("\nTests with results other than PASS that are expected:") + for test in sorted(expected): + issue_url = 'https://github.com/openzfs/zfs/issues/' + + # Include the reason why the result is expected, given the following: + # 1. Suppress test results which set the "Not applicable" reason. + # 2. Numerical reasons are assumed to be GitHub issue numbers. + # 3. When an entire test group is skipped only report the setup reason. + if test in known: + if known[test][1] == na_reason: + continue + elif known[test][1].isdigit(): + expect = issue_url + known[test][1] + else: + expect = known[test][1] + elif test in maybe: + if maybe[test][1].isdigit(): + expect = issue_url + maybe[test][1] + else: + expect = maybe[test][1] + elif setup in known and known[setup][0] == "SKIP" and setup != test: + continue + elif setup in maybe and maybe[setup][0] == "SKIP" and setup != test: + continue + else: + expect = "UNKNOWN REASON" + print(" %s %s (%s)" % (results[test], test, expect)) + + print("\nTests with result of PASS that are unexpected:") + for test in sorted(known.keys()): + # We probably should not be silently ignoring the case + # where "test" is not in "results". + if test not in results or results[test] != "PASS": + continue + print(" %s %s (expected %s)" % (results[test], test, + known[test][0])) + + print("\nTests with results other than PASS that are unexpected:") + for test in sorted(unexpected): + expect = "PASS" if test not in known else known[test][0] + print(" %s %s (expected %s)" % (results[test], test, expect)) + + if len(unexpected) == 0: + sys.exit(0) + else: + sys.exit(1) diff --git a/tests/test-runner/include/Makefile.am b/tests/test-runner/include/Makefile.am new file mode 100644 index 000000000000..d3eeb32de914 --- /dev/null +++ b/tests/test-runner/include/Makefile.am @@ -0,0 +1,5 @@ +pkgdatadir = $(datadir)/@PACKAGE@/test-runner/include + +dist_pkgdata_DATA = \ + logapi.shlib \ + stf.shlib diff --git a/tests/test-runner/include/logapi.shlib b/tests/test-runner/include/logapi.shlib new file mode 100644 index 000000000000..aa6e7c0f65fa --- /dev/null +++ b/tests/test-runner/include/logapi.shlib @@ -0,0 +1,530 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# Copyright (c) 2012, 2020 by Delphix. All rights reserved. +# + +. ${STF_TOOLS}/include/stf.shlib + +# Output an assertion +# +# $@ - assertion text + +function log_assert +{ + _printline ASSERTION: "$@" +} + +# Output a comment +# +# $@ - comment text + +function log_note +{ + _printline NOTE: "$@" +} + +# Execute and print command with status where success equals non-zero result +# +# $@ - command to execute +# +# return 0 if command fails, otherwise return 1 + +function log_neg +{ + log_neg_expect "" "$@" + return $? +} + +# Execute a positive test and exit $STF_FAIL is test fails +# +# $@ - command to execute + +function log_must +{ + log_pos "$@" + (( $? != 0 )) && log_fail +} + +# Execute a positive test but retry the command on failure if the output +# matches an expected pattern. Otherwise behave like log_must and exit +# $STF_FAIL is test fails. +# +# $1 - retry keyword +# $2 - retry attempts +# $3-$@ - command to execute +# +function log_must_retry +{ + typeset out="" + typeset logfile="/tmp/log.$$" + typeset status=1 + typeset expect=$1 + typeset retry=$2 + typeset delay=1 + shift 2 + + while [[ -e $logfile ]]; do + logfile="$logfile.$$" + done + + while (( $retry > 0 )); do + "$@" 2>$logfile + status=$? + out="cat $logfile" + + if (( $status == 0 )); then + $out | egrep -i "internal error|assertion failed" \ + > /dev/null 2>&1 + # internal error or assertion failed + if [[ $? -eq 0 ]]; then + print -u2 $($out) + _printerror "$@" "internal error or" \ + " assertion failure exited $status" + status=1 + else + [[ -n $LOGAPI_DEBUG ]] && print $($out) + _printsuccess "$@" + fi + break + else + $out | grep -i "$expect" > /dev/null 2>&1 + if (( $? == 0 )); then + print -u2 $($out) + _printerror "$@" "Retry in $delay seconds" + sleep $delay + + (( retry=retry - 1 )) + (( delay=delay * 2 )) + else + break; + fi + fi + done + + if (( $status != 0 )) ; then + print -u2 $($out) + _printerror "$@" "exited $status" + fi + + _recursive_output $logfile "false" + return $status +} + +# Execute a positive test and exit $STF_FAIL is test fails after being +# retried up to 5 times when the command returns the keyword "busy". +# +# $@ - command to execute +function log_must_busy +{ + log_must_retry "busy" 5 "$@" + (( $? != 0 )) && log_fail +} + +# Execute a negative test and exit $STF_FAIL if test passes +# +# $@ - command to execute + +function log_mustnot +{ + log_neg "$@" + (( $? != 0 )) && log_fail +} + +# Execute a negative test with keyword expected, and exit +# $STF_FAIL if test passes +# +# $1 - keyword expected +# $2-$@ - command to execute + +function log_mustnot_expect +{ + log_neg_expect "$@" + (( $? != 0 )) && log_fail +} + +# Signal numbers are platform-dependent +case $(uname) in +Darwin|FreeBSD) + SIGBUS=10 + SIGSEGV=11 + ;; +illumos|Linux|*) + SIGBUS=7 + SIGSEGV=11 + ;; +esac +EXIT_SUCCESS=0 +EXIT_NOTFOUND=127 +EXIT_SIGNAL=256 +EXIT_SIGBUS=$((EXIT_SIGNAL + SIGBUS)) +EXIT_SIGSEGV=$((EXIT_SIGNAL + SIGSEGV)) + +# Execute and print command with status where success equals non-zero result +# or output includes expected keyword +# +# $1 - keyword expected +# $2-$@ - command to execute +# +# return 0 if command fails, or the output contains the keyword expected, +# return 1 otherwise + +function log_neg_expect +{ + typeset out="" + typeset logfile="/tmp/log.$$" + typeset ret=1 + typeset expect=$1 + shift + + while [[ -e $logfile ]]; do + logfile="$logfile.$$" + done + + "$@" 2>$logfile + typeset status=$? + out="cat $logfile" + + # unexpected status + if (( $status == EXIT_SUCCESS )); then + print -u2 $($out) + _printerror "$@" "unexpectedly exited $status" + # missing binary + elif (( $status == EXIT_NOTFOUND )); then + print -u2 $($out) + _printerror "$@" "unexpectedly exited $status (File not found)" + # bus error - core dump + elif (( $status == EXIT_SIGBUS )); then + print -u2 $($out) + _printerror "$@" "unexpectedly exited $status (Bus Error)" + # segmentation violation - core dump + elif (( $status == EXIT_SIGSEGV )); then + print -u2 $($out) + _printerror "$@" "unexpectedly exited $status (SEGV)" + else + $out | egrep -i "internal error|assertion failed" \ + > /dev/null 2>&1 + # internal error or assertion failed + if (( $? == 0 )); then + print -u2 $($out) + _printerror "$@" "internal error or assertion failure" \ + " exited $status" + elif [[ -n $expect ]] ; then + $out | grep -i "$expect" > /dev/null 2>&1 + if (( $? == 0 )); then + ret=0 + else + print -u2 $($out) + _printerror "$@" "unexpectedly exited $status" + fi + else + ret=0 + fi + + if (( $ret == 0 )); then + [[ -n $LOGAPI_DEBUG ]] && print $($out) + _printsuccess "$@" "exited $status" + fi + fi + _recursive_output $logfile "false" + return $ret +} + +# Execute and print command with status where success equals zero result +# +# $@ command to execute +# +# return command exit status + +function log_pos +{ + typeset out="" + typeset logfile="/tmp/log.$$" + + while [[ -e $logfile ]]; do + logfile="$logfile.$$" + done + + "$@" 2>$logfile + typeset status=$? + out="cat $logfile" + + if (( $status != 0 )) ; then + print -u2 $($out) + _printerror "$@" "exited $status" + else + $out | egrep -i "internal error|assertion failed" \ + > /dev/null 2>&1 + # internal error or assertion failed + if [[ $? -eq 0 ]]; then + print -u2 $($out) + _printerror "$@" "internal error or assertion failure" \ + " exited $status" + status=1 + else + [[ -n $LOGAPI_DEBUG ]] && print $($out) + _printsuccess "$@" + fi + fi + _recursive_output $logfile "false" + return $status +} + +# Set an exit handler +# +# $@ - function(s) to perform on exit + +function log_onexit +{ + _CLEANUP=("$*") +} + +# Push an exit handler on the cleanup stack +# +# $@ - function(s) to perform on exit + +function log_onexit_push +{ + _CLEANUP+=("$*") +} + +# Pop an exit handler off the cleanup stack + +function log_onexit_pop +{ + _CLEANUP=("${_CLEANUP[@]:0:${#_CLEANUP[@]}-1}") +} + +# +# Exit functions +# + +# Perform cleanup and exit $STF_PASS +# +# $@ - message text + +function log_pass +{ + _endlog $STF_PASS "$@" +} + +# Perform cleanup and exit $STF_FAIL +# +# $@ - message text + +function log_fail +{ + _endlog $STF_FAIL "$@" +} + +# Perform cleanup and exit $STF_UNRESOLVED +# +# $@ - message text + +function log_unresolved +{ + _endlog $STF_UNRESOLVED "$@" +} + +# Perform cleanup and exit $STF_NOTINUSE +# +# $@ - message text + +function log_notinuse +{ + _endlog $STF_NOTINUSE "$@" +} + +# Perform cleanup and exit $STF_UNSUPPORTED +# +# $@ - message text + +function log_unsupported +{ + _endlog $STF_UNSUPPORTED "$@" +} + +# Perform cleanup and exit $STF_UNTESTED +# +# $@ - message text + +function log_untested +{ + _endlog $STF_UNTESTED "$@" +} + +# Perform cleanup and exit $STF_UNINITIATED +# +# $@ - message text + +function log_uninitiated +{ + _endlog $STF_UNINITIATED "$@" +} + +# Perform cleanup and exit $STF_NORESULT +# +# $@ - message text + +function log_noresult +{ + _endlog $STF_NORESULT "$@" +} + +# Perform cleanup and exit $STF_WARNING +# +# $@ - message text + +function log_warning +{ + _endlog $STF_WARNING "$@" +} + +# Perform cleanup and exit $STF_TIMED_OUT +# +# $@ - message text + +function log_timed_out +{ + _endlog $STF_TIMED_OUT "$@" +} + +# Perform cleanup and exit $STF_OTHER +# +# $@ - message text + +function log_other +{ + _endlog $STF_OTHER "$@" +} + +function set_main_pid +{ + _MAINPID=$1 +} + +# +# Internal functions +# + +# Execute custom callback scripts on test failure +# +# callback script paths are stored in TESTFAIL_CALLBACKS, delimited by ':'. + +function _execute_testfail_callbacks +{ + typeset callback + + print "$TESTFAIL_CALLBACKS:" | while read -d ":" callback; do + if [[ -n "$callback" ]] ; then + log_note "Performing test-fail callback ($callback)" + $callback + fi + done +} + +# Perform cleanup and exit +# +# $1 - stf exit code +# $2-$n - message text + +function _endlog +{ + typeset logfile="/tmp/log.$$" + _recursive_output $logfile + + typeset exitcode=$1 + shift + (( ${#@} > 0 )) && _printline "$@" + + # + # If we're running in a subshell then just exit and let + # the parent handle the failures + # + if [[ -n "$_MAINPID" && $$ != "$_MAINPID" ]]; then + log_note "subshell exited: "$_MAINPID + exit $exitcode + fi + + if [[ $exitcode == $STF_FAIL ]] ; then + _execute_testfail_callbacks + fi + + typeset stack=("${_CLEANUP[@]}") + log_onexit "" + typeset i=${#stack[@]} + while (( i-- )); do + typeset cleanup="${stack[i]}" + log_note "Performing local cleanup via log_onexit ($cleanup)" + $cleanup + done + + exit $exitcode +} + +# Output a formatted line +# +# $@ - message text + +function _printline +{ + print "$@" +} + +# Output an error message +# +# $@ - message text + +function _printerror +{ + _printline ERROR: "$@" +} + +# Output a success message +# +# $@ - message text + +function _printsuccess +{ + _printline SUCCESS: "$@" +} + +# Output logfiles recursively +# +# $1 - start file +# $2 - indicate whether output the start file itself, default as yes. + +function _recursive_output #logfile +{ + typeset logfile=$1 + + while [[ -e $logfile ]]; do + if [[ -z $2 || $logfile != $1 ]]; then + cat $logfile + fi + rm -f $logfile + logfile="$logfile.$$" + done +} diff --git a/tests/test-runner/include/stf.shlib b/tests/test-runner/include/stf.shlib new file mode 100644 index 000000000000..ea879a84c131 --- /dev/null +++ b/tests/test-runner/include/stf.shlib @@ -0,0 +1,57 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# Copyright (c) 2012 by Delphix. All rights reserved. +# + + +STF_PASS=0 +STF_FAIL=1 +STF_UNRESOLVED=2 +STF_NOTINUSE=3 +STF_UNSUPPORTED=4 +STF_UNTESTED=5 +STF_UNINITIATED=6 +STF_NORESULT=7 +STF_WARNING=8 +STF_TIMED_OUT=9 +STF_OTHER=10 + +# do this to use the names: eval echo \$STF_RESULT_NAME_${result} +STF_RESULT_NAME_0="PASS" +STF_RESULT_NAME_1="FAIL" +STF_RESULT_NAME_2="UNRESOLVED" +STF_RESULT_NAME_3="NOTINUSE" +STF_RESULT_NAME_4="UNSUPPORTED" +STF_RESULT_NAME_5="UNTESTED" +STF_RESULT_NAME_6="UNINITIATED" +STF_RESULT_NAME_7="NORESULT" +STF_RESULT_NAME_8="WARNING" +STF_RESULT_NAME_9="TIMED_OUT" +STF_RESULT_NAME_10="OTHER" + +# do this to use the array: ${STF_RESULT_NAMES[$result]} +STF_RESULT_NAMES=( "PASS" "FAIL" "UNRESOLVED" "NOTINUSE" "UNSUPPORTED" \ + "UNTESTED" "UNINITIATED" "NORESULT" "WARNING" "TIMED_OUT" "OTHER" ) diff --git a/tests/test-runner/man/Makefile.am b/tests/test-runner/man/Makefile.am new file mode 100644 index 000000000000..a7017f5f0535 --- /dev/null +++ b/tests/test-runner/man/Makefile.am @@ -0,0 +1,4 @@ +dist_man_MANS = test-runner.1 + +install-data-local: + $(INSTALL) -d -m 0755 "$(DESTDIR)$(mandir)/man1" diff --git a/tests/test-runner/man/test-runner.1 b/tests/test-runner/man/test-runner.1 new file mode 100644 index 000000000000..19c636c8cb7c --- /dev/null +++ b/tests/test-runner/man/test-runner.1 @@ -0,0 +1,386 @@ +.\" +.\" This file and its contents are supplied under the terms of the +.\" Common Development and Distribution License ("CDDL"), version 1.0. +.\" You may only use this file in accordance with the terms of version +.\" 1.0 of the CDDL. +.\" +.\" A full copy of the text of the CDDL should have accompanied this +.\" source. A copy of the CDDL is also available via the Internet at +.\" http://www.illumos.org/license/CDDL. +.\" +.\" +.\" Copyright (c) 2012 by Delphix. All rights reserved. +.\" +.TH run 1 "23 Sep 2012" +.SH NAME +run \- find, execute, and log the results of tests +.SH SYNOPSIS +.LP +.nf +\fBrun\fR [\fB-dgq] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR] [\fB-uxX\fR \fIusername\fR] + \fIpathname\fR ... +.fi + +.LP +.nf +\fBrun\fR \fB-w\fR \fIrunfile\fR [\fB-gq\fR] [\fB-o\fR \fIoutputdir\fR] [\fB-pP\fR \fIscript\fR] [\fB-t\fR \fIseconds\fR] + [\fB-uxX\fR \fIusername\fR] \fIpathname\fR ... +.fi + +.LP +.nf +\fBrun\fR \fB-c\fR \fIrunfile\fR [\fB-dq\fR] +.fi + +.LP +.nf +\fBrun\fR [\fB-h\fR] +.fi + +.SH DESCRIPTION +.sp +.LP +The \fBrun\fR command has three basic modes of operation. With neither the +\fB-c\fR nor the \fB-w\fR option, \fBrun\fR processes the arguments provided on +the command line, adding them to the list for this run. If a specified +\fIpathname\fR is an executable file, it is added as a test. If a specified +\fIpathname\fR is a directory, the behavior depends upon the \fB-g\fR option. +If \fB-g\fR is specified, the directory is treated as a test group. See the +section on "Test Groups" below. Without the \fB-g\fR option, \fBrun\fR simply +descends into the directory looking for executable files. The tests are then +executed, and the results are logged. + +With the \fB-w\fR option, \fBrun\fR finds tests in the manner described above. +Rather than executing the tests and logging the results, the test configuration +is stored in a \fIrunfile\fR which can be used in future invocations, or edited +to modify which tests are executed and which options are applied. Options +included on the command line with \fB-w\fR become defaults in the +\fIrunfile\fR. + +With the \fB-c\fR option, \fBrun\fR parses a \fIrunfile\fR, which can specify a +series of tests and test groups to be executed. The tests are then executed, +and the results are logged. +.sp +.SS "Test Groups" +.sp +.LP +A test group is comprised of a set of executable files, all of which exist in +one directory. The options specified on the command line or in a \fIrunfile\fR +apply to individual tests in the group. The exception is options pertaining to +pre and post scripts, which act on all tests as a group. Rather than running +before and after each test, these scripts are run only once each at the start +and end of the test group. +.SS "Test Execution" +.sp +.LP +The specified tests run serially, and are typically assigned results according +to exit values. Tests that exit zero and non-zero are marked "PASS" and "FAIL" +respectively. When a pre script fails for a test group, only the post script is +executed, and the remaining tests are marked "SKIPPED." Any test that exceeds +its \fItimeout\fR is terminated, and marked "KILLED." + +By default, tests are executed with the credentials of the \fBrun\fR script. +Executing tests with other credentials is done via \fBsudo\fR(1m), which must +be configured to allow execution without prompting for a password. Environment +variables from the calling shell are available to individual tests. During test +execution, the working directory is changed to \fIoutputdir\fR. +.SS "Output Logging" +.sp +.LP +By default, \fBrun\fR will print one line on standard output at the conclusion +of each test indicating the test name, result and elapsed time. Additionally, +for each invocation of \fBrun\fR, a directory is created using the ISO 8601 +date format. Within this directory is a file named \fIlog\fR containing all the +test output with timestamps, and a directory for each test. Within the test +directories, there is one file each for standard output, standard error and +merged output. The default location for the \fIoutputdir\fR is +\fI/var/tmp/test_results\fR. +.SS "Runfiles" +.sp +.LP +The \fIrunfile\fR is an ini style configuration file that describes a test run. +The file has one section named "DEFAULT," which contains configuration option +names and their values in "name = value" format. The values in this section +apply to all the subsequent sections, unless they are also specified there, in +which case the default is overridden. The remaining section names are the +absolute pathnames of files and directories, describing tests and test groups +respectively. The legal option names are: +.sp +.ne 2 +.na +\fBoutputdir\fR = \fIpathname\fR +.ad +.sp .6 +.RS 4n +The name of the directory that holds test logs. +.RE +.sp +.ne 2 +.na +\fBpre\fR = \fIscript\fR +.ad +.sp .6 +.RS 4n +Run \fIscript\fR prior to the test or test group. +.RE +.sp +.ne 2 +.na +\fBpre_user\fR = \fIusername\fR +.ad +.sp .6 +.RS 4n +Execute the pre script as \fIusername\fR. +.RE +.sp +.ne 2 +.na +\fBpost\fR = \fIscript\fR +.ad +.sp .6 +.RS 4n +Run \fIscript\fR after the test or test group. +.RE +.sp +.ne 2 +.na +\fBpost_user\fR = \fIusername\fR +.ad +.sp .6 +.RS 4n +Execute the post script as \fIusername\fR. +.RE +.sp +.ne 2 +.na +\fBquiet\fR = [\fITrue\fR|\fIFalse\fR] +.ad +.sp .6 +.RS 4n +If set to True, only the results summary is printed to standard out. +.RE +.sp +.ne 2 +.na +\fBtests\fR = [\fI'filename'\fR [,...]] +.ad +.sp .6 +.RS 4n +Specify a list of \fIfilenames\fR for this test group. Only the basename of the +absolute path is required. This option is only valid for test groups, and each +\fIfilename\fR must be single quoted. +.RE +.sp +.ne 2 +.na +\fBtimeout\fR = \fIn\fR +.ad +.sp .6 +.RS 4n +A timeout value of \fIn\fR seconds. +.RE +.sp +.ne 2 +.na +\fBuser\fR = \fIusername\fR +.ad +.sp .6 +.RS 4n +Execute the test or test group as \fIusername\fR. +.RE + +.SH OPTIONS +.sp +.LP +The following options are available for the \fBrun\fR command. +.sp +.ne 2 +.na +\fB-c\fR \fIrunfile\fR +.ad +.RS 6n +Specify a \fIrunfile\fR to be consumed by the run command. +.RE + +.ne 2 +.na +\fB-d\fR +.ad +.RS 6n +Dry run mode. Execute no tests, but print a description of each test that would +have been run. +.RE + +.ne 2 +.na +\fB-g\fR +.ad +.RS 6n +Create test groups from any directories found while searching for tests. +.RE + +.ne 2 +.na +\fB-o\fR \fIoutputdir\fR +.ad +.RS 6n +Specify the directory in which to write test results. +.RE + +.ne 2 +.na +\fB-p\fR \fIscript\fR +.ad +.RS 6n +Run \fIscript\fR prior to any test or test group. +.RE + +.ne 2 +.na +\fB-P\fR \fIscript\fR +.ad +.RS 6n +Run \fIscript\fR after any test or test group. +.RE + +.ne 2 +.na +\fB-q\fR +.ad +.RS 6n +Print only the results summary to the standard output. +.RE + +.ne 2 +.na +\fB-s\fR \fIscript\fR +.ad +.RS 6n +Run \fIscript\fR as a failsafe after any test is killed. +.RE + +.ne 2 +.na +\fB-S\fR \fIusername\fR +.ad +.RS 6n +Execute the failsafe script as \fIusername\fR. +.RE + +.ne 2 +.na +\fB-t\fR \fIn\fR +.ad +.RS 6n +Specify a timeout value of \fIn\fR seconds per test. +.RE + +.ne 2 +.na +\fB-u\fR \fIusername\fR +.ad +.RS 6n +Execute tests or test groups as \fIusername\fR. +.RE + +.ne 2 +.na +\fB-w\fR \fIrunfile\fR +.ad +.RS 6n +Specify the name of the \fIrunfile\fR to create. +.RE + +.ne 2 +.na +\fB-x\fR \fIusername\fR +.ad +.RS 6n +Execute the pre script as \fIusername\fR. +.RE + +.ne 2 +.na +\fB-X\fR \fIusername\fR +.ad +.RS 6n +Execute the post script as \fIusername\fR. +.RE + +.SH EXAMPLES +.LP +\fBExample 1\fR Running ad-hoc tests. +.sp +.LP +This example demonstrates the simplest invocation of \fBrun\fR. + +.sp +.in +2 +.nf +% \fBrun my-tests\fR +Test: /home/jkennedy/my-tests/test-01 [00:02] [PASS] +Test: /home/jkennedy/my-tests/test-02 [00:04] [PASS] +Test: /home/jkennedy/my-tests/test-03 [00:01] [PASS] + +Results Summary +PASS 3 + +Running Time: 00:00:07 +Percent passed: 100.0% +Log directory: /var/tmp/test_results/20120923T180654 +.fi +.in -2 + +.LP +\fBExample 2\fR Creating a \fIrunfile\fR for future use. +.sp +.LP +This example demonstrates creating a \fIrunfile\fR with non default options. + +.sp +.in +2 +.nf +% \fBrun -p setup -x root -g -w new-tests.run new-tests\fR +% \fBcat new-tests.run\fR +[DEFAULT] +pre = setup +post_user = +quiet = False +user = +timeout = 60 +post = +pre_user = root +outputdir = /var/tmp/test_results + +[/home/jkennedy/new-tests] +tests = ['test-01', 'test-02', 'test-03'] +.fi +.in -2 + +.SH EXIT STATUS +.sp +.LP +The following exit values are returned: +.sp +.ne 2 +.na +\fB\fB0\fR\fR +.ad +.sp .6 +.RS 4n +Successful completion. +.RE +.sp +.ne 2 +.na +\fB\fB1\fR\fR +.ad +.sp .6 +.RS 4n +An error occurred. +.RE + +.SH SEE ALSO +.sp +.LP +\fBsudo\fR(1m) |