diff options
author | Ed Maste <emaste@FreeBSD.org> | 2019-01-10 14:18:11 +0000 |
---|---|---|
committer | Ed Maste <emaste@FreeBSD.org> | 2019-01-10 14:18:11 +0000 |
commit | 2b92b30119ed91ed88f102ba9ecc40cd1c046a65 (patch) | |
tree | 5cde2d0340810533c039044df9508aee4fd482b3 /test/libtest | |
parent | 87ec209e33f42bd708452228ae84ded370b70163 (diff) |
Notes
Diffstat (limited to 'test/libtest')
-rw-r--r-- | test/libtest/Makefile | 17 | ||||
-rw-r--r-- | test/libtest/README.rst | 116 | ||||
-rw-r--r-- | test/libtest/bin/Makefile | 9 | ||||
-rwxr-xr-x | test/libtest/bin/make-test-scaffolding | 213 | ||||
-rw-r--r-- | test/libtest/bin/make-test-scaffolding.1 | 111 | ||||
-rw-r--r-- | test/libtest/driver/Makefile | 14 | ||||
-rw-r--r-- | test/libtest/driver/test_main.c | 49 | ||||
-rw-r--r-- | test/libtest/examples/Makefile | 14 | ||||
-rw-r--r-- | test/libtest/examples/minimal_example.c | 50 | ||||
-rw-r--r-- | test/libtest/examples/simple_example.c | 145 | ||||
-rw-r--r-- | test/libtest/lib/Makefile | 16 | ||||
-rw-r--r-- | test/libtest/lib/test.3 | 110 | ||||
-rw-r--r-- | test/libtest/lib/test.c | 29 | ||||
-rw-r--r-- | test/libtest/lib/test.h | 159 | ||||
-rw-r--r-- | test/libtest/lib/test_runner.c | 31 | ||||
-rw-r--r-- | test/libtest/lib/test_runner.h | 118 |
16 files changed, 1201 insertions, 0 deletions
diff --git a/test/libtest/Makefile b/test/libtest/Makefile new file mode 100644 index 0000000000000..1dc489b9db8a7 --- /dev/null +++ b/test/libtest/Makefile @@ -0,0 +1,17 @@ +# $Id$ +# +# The test(3) API. + +TOP= ../.. + +SUBDIR+= bin +SUBDIR+= lib +SUBDIR+= driver +SUBDIR+= examples + +.if !make(install) +.include "$(TOP)/mk/elftoolchain.subdir.mk" +.else +install: .SILENT .PHONY + echo Nothing to install. +.endif diff --git a/test/libtest/README.rst b/test/libtest/README.rst new file mode 100644 index 0000000000000..3f29c85f8a93d --- /dev/null +++ b/test/libtest/README.rst @@ -0,0 +1,116 @@ +===================================== +test(3) - a library for writing tests +===================================== + +The ``test(3)`` API and its related scaffolding generator +(`make-test-scaffolding(1) <mts_>`_) work together to reduce the +boilerplate needed for tests. + +.. _mts: bin/make-test-scaffolding + +Quick Start +=========== + +The following source code defines a test suite that contains a single +test: + +.. code:: c + + /* File: test.c */ + #include "test.h" + + enum test_result + tf_goodbye_world(testcase_state tcs) + { + return (TEST_PASS); + } + +By convention, test functions are named using a ``tf_`` prefix. + +Given an object file compiled from this source, the +`make-test-scaffolding(1) <mts_>`_ utility would generate scaffolding +describing a single invocable test named "``goodbye_world``". + +Test Cases +---------- + +Test functions that are related to each other can be grouped into test +cases. The following code snippet defines a test suite with two test +functions contained in a test case named "``helloworld``": + +.. code:: c + + /* File: test.c */ + #include "test.h" + + TESTCASE_DESCRIPTION(helloworld) = + "A description of the helloworld test case."; + + enum test_result + tf_helloworld_hello(testcase_state tcs) + { + return (TEST_PASS); + } + + enum test_result + tf_helloworld_goodbye(testcase_state tcs) + { + return (TEST_FAIL); + } + +Test cases can define their own set up and tear down functions: + +.. code:: c + + /* File: test.c continued. */ + struct helloworld_test { .. state used by the helloworld tests .. }; + + enum testcase_status + tc_setup_helloworld(testcase_state *tcs) + { + *tcs = ..allocate a struct helloworld_test.. ; + return (TESTCASE_OK); + } + + enum testcase_status + tc_teardown_helloworld(testcase_state tcs) + { + .. deallocate test case state.. + return (TESTCASE_OK); + } + +The set up function for a test case will be invoked prior to any of +the functions that are part of the test case. The set up function can +allocate test-specific state, which is then passed to each test function +for its use. + +The tear down function for a test case will be invoked after the test +functions in the test case are invoked. This function is responsible for +deallocating the resources allocated by its corresponding set up function. + +Building Tests +-------------- + +Within the `Elftoolchain Project`_'s sources, the ``elftoolchain.test.mk`` +rule set handles the process of invoking the `make-test-scaffolding(1) +<mts_>`_ utility and building an test executable. + +.. code:: make + + # Example Makefile. + + TOP= ..path to the top of the elftoolchain source tree.. + + TEST_SRCS= test.c + + .include "$(TOP)/mk/elftoolchain.test.mk" + + +.. _Elftoolchain Project: http://elftoolchain.sourceforge.net/ + +Further Reading +=============== + +- The `test(3) <lib/test.3>`_ manual page. +- The `make-test-scaffolding(1) <bin/make-test-scaffolding.1>`_ manual page. +- `Example code <examples/>`_. diff --git a/test/libtest/bin/Makefile b/test/libtest/bin/Makefile new file mode 100644 index 0000000000000..a6a3b8d6ae43d --- /dev/null +++ b/test/libtest/bin/Makefile @@ -0,0 +1,9 @@ +# $Id$ + +TOP= ../../.. + +SCRIPTS= make-test-scaffolding + +MAN1= make-test-scaffolding.1 + +.include "${TOP}/mk/elftoolchain.prog.mk" diff --git a/test/libtest/bin/make-test-scaffolding b/test/libtest/bin/make-test-scaffolding new file mode 100755 index 0000000000000..c0966d34527b3 --- /dev/null +++ b/test/libtest/bin/make-test-scaffolding @@ -0,0 +1,213 @@ +#!/bin/sh +# +# Copyright (c) 2018, Joseph Koshy. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer +# in this position and unchanged. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# $Id$ + +# Given a list of objects that use the test(3) API, this script will +# generate test case and test function descriptors based on the symbols +# contained in those objects. + +usage() +{ + echo Usage: `basename $0`: "[options] objects..." + echo + echo "Generate test(3) scaffolding from objects." + echo "Options include:" + echo + echo "\t-o out\t\tcreate output file \"out\" [default \"tc.c\"]." + echo +} + +output_file="tc.c" +prefix_tc_descr='tc_description_' +prefix_tc_setup='tc_setup_' +prefix_tc_tags='tc_tags_' +prefix_tc_teardown='tc_teardown_' +prefix_tf='tf_' +prefix_tf_descr='tf_description_' +prefix_tf_tags='tf_tags_' + + +args=`getopt o: $*` +if [ ${?} -ne 0 ]; then + usage + exit 2 +fi + +set -- ${args} + +for i +do + case "${i}" in + -o ) + output_file="${2}" + shift; shift;; + -- ) + shift; break;; + esac +done + +if [ ${#} -eq 0 ]; then + usage + exit 2 +fi + +exec > ${output_file} +cat <<EOF +/* GENERATED FROM: ${@} */ +#include <stddef.h> +#include "test.h" +#include "test_runner.h" +EOF + +if ! nm ${*} | sort -k 3 | \ + awk -v prefix_tc_descr=${prefix_tc_descr} \ + -v prefix_tc_setup=${prefix_tc_setup} \ + -v prefix_tc_tags=${prefix_tc_tags} \ + -v prefix_tc_teardown=${prefix_tc_teardown} \ + -v prefix_tf=${prefix_tf} \ + -v prefix_tf_descr=${prefix_tf_descr} \ + -v prefix_tf_tags=${prefix_tf_tags} ' + function suffix(value, prefix) { + return substr(value, length(prefix) + 1); + } + function matched_test_case(tf_name, tc_matched) { + tc_matched = "" + for (tc_name in test_cases) { + if (tf_name ~ tc_name "_" && + length(tc_name) > length(tc_matched)) { + tc_matched = tc_name + } + } + if (tc_matched != "") + return tc_matched + return DEFAULT + } + function print_test_case_record(tc_name) { + printf("\t{\n") + printf("\t\t.tc_name = \"%s\",\n", tc_name) + printf("\t\t.tc_description = %s,\n", test_case_descriptions[tc_name]) + printf("\t\t.tc_tags = %s,\n", test_case_tags[tc_name]) + printf("\t\t.tc_tests = test_functions_%s\n", tc_name) + printf("\t},\n") + } + function delete_test_functions(tc_name) { + for (tf_name in test_functions) { + if (matched_test_case(tf_name) == tc_name) + delete test_functions[tf_name] + } + } + function print_test_functions_record(tc_name) { + printf("struct test_descriptor test_functions_%s[] = {\n", tc_name) + for (tf_name in test_functions) { + if (tc_name != matched_test_case(tf_name)) + continue + printf("\t{\n") + printf("\t\t.t_name = \"%s\",\n", tf_name) + printf("\t\t.t_description = %s,\n", + test_function_descriptions[tf_name]) + printf("\t\t.t_func = %s,\n", prefix_tf tf_name) + printf("\t\t.t_tags = %s\n", test_function_tags[tf_name]) + printf("\t},\n") + } + printf("};\n") + } + function is_non_empty(array, i) { + for (i in array) return 1 + return 0 + } + BEGIN { + DEFAULT = "default" + test_case_descriptions[DEFAULT] = "NULL" + test_case_tags[DEFAULT] = "NULL" + } + ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tc_descr { + printf("extern testcase_description %s;\n", $3) + tc_name = suffix($3, prefix_tc_descr) + test_cases[tc_name] = 1 + test_case_descriptions[tc_name] = $3 + } + $2 == "T" && $3 ~ "^" prefix_tc_setup { + tc_name = suffix($3, prefix_tc_setup) + test_cases[tc_name] = 1 + test_case_setup[tc_name] = $3 + } + ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tc_tags { + printf("extern testcase_tags %s;\n", $3) + tc_name = suffix($3, prefix_tc_tags) + test_cases[tc_name] = 1 + test_case_tags[tc_name] = $3 + } + $2 == "T" && $3 ~ "^" prefix_tc_teardown { + tc_name = suffix($3, prefix_tc_teardown) + test_cases[tc_name] = 1 + test_case_teardown[tc_name] = $3 + } + ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tf_descr { + printf("extern test_description %s;\n", $3) + tf_name = suffix($3, prefix_tf_descr) + test_function_descriptions[tf_name] = $3 + } + ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tf_tags { + printf("extern test_tags %s;\n", $3) + tf_name = suffix($3, prefix_tf_tags) + test_function_tags[tf_name] = $3 + } + $2 == "T" && $3 ~ "^" prefix_tf { + printf("test_function %s;\n", $3) + tf_name = suffix($3, prefix_tf) + test_functions[tf_name] = 1 + } + END { + for (tf_name in test_functions) { + if (test_function_descriptions[tf_name] == "") + test_function_descriptions[tf_name] = "NULL" + if (test_function_tags[tf_name] == "") + test_function_tags[tf_name] = "NULL" + } + for (tc_name in test_cases) { + if (test_case_descriptions[tc_name] == "") + test_case_descriptions[tc_name] = "NULL" + if (test_case_tags[tc_name] == "") + test_case_tags[tc_name] = "NULL" + } + for (tc_name in test_cases) { + print_test_functions_record(tc_name) + delete_test_functions(tc_name) + } + needs_default = is_non_empty(test_functions) + if (needs_default) + print_test_functions_record(DEFAULT) + printf("struct test_case_descriptor test_cases[] = {\n") + for (tc_name in test_cases) + print_test_case_record(tc_name) + if (needs_default) + print_test_case_record(DEFAULT) + printf("};\n") + }'; then + # Cleanup in case of an error. + rm ${output_file} + exit 1 +fi diff --git a/test/libtest/bin/make-test-scaffolding.1 b/test/libtest/bin/make-test-scaffolding.1 new file mode 100644 index 0000000000000..2d120e1bb5b21 --- /dev/null +++ b/test/libtest/bin/make-test-scaffolding.1 @@ -0,0 +1,111 @@ +.\" Copyright (c) 2018, Joseph Koshy. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" This software is provided by Joseph Koshy ``as is'' and +.\" any express or implied warranties, including, but not limited to, the +.\" implied warranties of merchantability and fitness for a particular purpose +.\" are disclaimed. in no event shall Joseph Koshy be liable +.\" for any direct, indirect, incidental, special, exemplary, or consequential +.\" damages (including, but not limited to, procurement of substitute goods +.\" or services; loss of use, data, or profits; or business interruption) +.\" however caused and on any theory of liability, whether in contract, strict +.\" liability, or tort (including negligence or otherwise) arising in any way +.\" out of the use of this software, even if advised of the possibility of +.\" such damage. +.\" +.\" $Id$ +.\" +.Dd December 25, 2018 +.Dt TEST 1 +.Os +.Sh NAME +.Nm make-test-scaffolding +.Nd generate scaffolding for tests +.Sh SYNOPSIS +.Nm +.Op Fl o Ar output_file +.Ar test_object ... +.Sh DESCRIPTION +The +.Nm +utility takes as input object files containing symbols following +its naming convention and generates the scaffolding needed to assemble +a test executable. +.Pp +The +.Nm +utility expects symbols to be named using the following conventions: +.Bl -bullet +.It +Test case descriptions are named using the +.Li tc_description_ +prefix, followed by the name of the test case. +For example, the symbol +.Sy tc_description_helloworld +would name the description of a test case named +.Dq helloworld . +.It +Test set up functions are named using a +.Li tc_setup_ +prefix, followed by the name of a test case. +For example, the function +.Fn tc_setup_helloworld +would name the set up function for a test case named +.Dq helloworld . +.It +Test tear down functions are named using a +.Li tc_teardown_ +prefix, followed by the name of a test case. +For example, the function +.Fn tc_teardown_helloworld +would name the tear down function for a test case named +.Dq helloworld . +.It +Tags for test cases are named using symbols with a +.Li tc_tags_ +prefix, followed by the name of their test case. +.It +Test functions are named using a +.Li tf_ +prefix, followed by the name for their containing test case and a +disambiguating suffix. +For example, the two test functions +.Fn tf_helloworld_sayhello +and +.Fn tf_helloworld_saygoodbye +would be associated with the test case named +.Dq helloworld . +.Pp +A test function that is not associated with a test case will be +added to a special test case named +.Dq default . +.It +Tags for test functions are named using symbols with a +.Li tf_tags_ +prefix, followed by the name of their test case. +.El +.Pp +The +.Nm +utility will generate a C source file containing the appropriate +.Vt "struct test_descriptor" +and +.Vt "struct test_case_descriptor" +structures for use by a test driver. +.Sh SEE ALSO +.Xr test 3 , +.Xr test_runner 3 +.Sh AUTHORS +The +.Nm +utility was written by +.An Joseph Koshy Aq Mt jkoshy@users.sourceforge.net . diff --git a/test/libtest/driver/Makefile b/test/libtest/driver/Makefile new file mode 100644 index 0000000000000..d149fee159d4a --- /dev/null +++ b/test/libtest/driver/Makefile @@ -0,0 +1,14 @@ +# $Id$ +# +# A command-line driver for libtest based tests. + +TOP= ../../.. + +CFLAGS+= -I${TOP}/test/libtest/lib + +LIB= test_main +SRCS= test_main.c + +WARNS?= 6 + +.include "$(TOP)/mk/elftoolchain.lib.mk" diff --git a/test/libtest/driver/test_main.c b/test/libtest/driver/test_main.c new file mode 100644 index 0000000000000..90e46d9a5fe24 --- /dev/null +++ b/test/libtest/driver/test_main.c @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This file defines a "main" that parses command-line arguments and invokes + * the selected test cases. + */ +#include <sys/param.h> + +#include <assert.h> +#include <stdlib.h> + +#include "_elftc.h" +#include "test.h" +#include "test_runner.h" + +ELFTC_VCSID("$Id$"); + +int +main(int argc, char **argv) +{ + (void) test_cases; + (void) argc; + (void) argv; + exit(0); +} diff --git a/test/libtest/examples/Makefile b/test/libtest/examples/Makefile new file mode 100644 index 0000000000000..9cc82d3dd7a4d --- /dev/null +++ b/test/libtest/examples/Makefile @@ -0,0 +1,14 @@ +# $Id$ +# +# Examples of tests built using test(3). + +TOP= ../../.. + +NOMAN= true + +CFLAGS+= -I../lib -Wall -pedantic + +TEST_SRCS= minimal_example.c \ + simple_example.c + +.include "$(TOP)/mk/elftoolchain.test.mk" diff --git a/test/libtest/examples/minimal_example.c b/test/libtest/examples/minimal_example.c new file mode 100644 index 0000000000000..4ad08b4bce06d --- /dev/null +++ b/test/libtest/examples/minimal_example.c @@ -0,0 +1,50 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $Id$ */ + +/* + * This C source defines a single test function named 'tf_helloworld', + * that is not part of a test case, and lacking a description or tags. + * + * Since no test cases are specified in this file, the test function + * would be assigned to the default test case (named 'default'). + * + * Given the object file created from this source, the + * 'make-test-scaffolding' utility will prepare the scaffolding + * needed to build an executable containing the test function. + */ + +#include "test.h" + +/* + * Function names prefixed with 'tf_' name test functions. + */ +enum test_result +tf_helloworld(testcase_state state) +{ + return (TEST_PASS); +} diff --git a/test/libtest/examples/simple_example.c b/test/libtest/examples/simple_example.c new file mode 100644 index 0000000000000..6a4f6697eb51e --- /dev/null +++ b/test/libtest/examples/simple_example.c @@ -0,0 +1,145 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $Id$ */ + +#include <stddef.h> + +#include "test.h" + +/* + * This source defines a single test case named 'helloworld' containing a + * single test function named 'sayhello' contained in that test case. At + * test execution time the test case would be selectable using the tags + * "tag1" or "tag2", or by its name 'helloworld'. The test function can + * be selected using tags "tag3" or "tag4", or by its name + * 'helloworld_sayhello'. + * + * Given the object code generated from this file, the + * 'make-test-scaffolding' utility will prepare the scaffolding needed + * to create a executable that can be used to execute these tests. + * + * Specifically the 'make-test-scaffolding' utilit will generate test and + * test case descriptors equivalent to: + * + * struct test_descriptor test_functions_helloworld[] = { + * { + * .t_description = tf_description_helloworld_sayhello, + * .t_tags = tf_tags_helloworld_sayhello, + * .t_func = tf_helloworld_sayhello + * } + * }; + * + * struct test_case_descriptor test_cases[] = { + * { + * .tc_description = tc_description_helloworld, + * .tc_tags = tc_tags_helloworld, + * .tc_tests = test_functions_helloworld + * } + * }; + */ + +/* + * A symbol name prefixed with 'tc_description_' contains a + * test case description. The TESTCASE_DESCRIPTION macro offers + * a convenient way to define such symbols. In the case of the + * symbol below, the test case named is 'helloworld'. + */ +TESTCASE_DESCRIPTION(helloworld) = "A description for a test case."; + +/* + * Function names prefixed with 'tc_setup_' are assumed to be test + * case set up functions. + */ +enum testcase_status +tc_setup_helloworld(testcase_state *state) +{ + return (TESTCASE_OK); +} + +/* + * Function names prefixed with 'tc_teardown_' are assumed to be test + * case tear down functions. + */ +enum testcase_status +tc_teardown_helloworld(testcase_state state) +{ + return (TESTCASE_OK); +} + +/* + * Names prefixed with 'tc_tags_' denote the tags associated with test + * cases. The TESTCASE_TAGS macro offers a convenient way to define such + * symbols. + * + * In the example below, all test functions belonging to the test case + * named 'helloworld' would be associated with tags "tag1" and "tag2". + * + * Tags lists are terminated by a NULL entry. + */ +TESTCASE_TAGS(helloworld) = { + "tag1", + "tag2", + NULL +}; + +/* + * Function names prefixed with 'tf_' name test functions. + */ +enum test_result +tf_helloworld_sayhello(testcase_state state) +{ + return (TEST_PASS); +} + +/* + * Names prefixed by 'tf_description_' contain descriptions of test + * functions (e.g., 'tf_description_helloworld_sayhello' contains the + * description for test function 'tf_helloworld_sayhello'). + * + * The TEST_DESCRIPTION macro offers a convenient way to define such + * symbols. + */ +TEST_DESCRIPTION(helloworld_sayhello) = + "A description for the test function 'tf_helloworld_sayhello'."; + +/* + * Names prefixed by 'tf_tags_' contain the tags associated with + * test functions. + * + * In the example below, the tags 'tag3' and 'tag4' are associated + * with the test function 'tf_helloworld_sayhello'. + * + * Alternately, the TEST_TAGS() macro offers a convenient way to + * define such symbols. + * + * Tags lists are terminated by a NULL entry. + */ +test_tags tf_tags_helloworld_sayhello = { + "tag3", + "tag4", + NULL +}; diff --git a/test/libtest/lib/Makefile b/test/libtest/lib/Makefile new file mode 100644 index 0000000000000..7359ddd76a6c7 --- /dev/null +++ b/test/libtest/lib/Makefile @@ -0,0 +1,16 @@ +# $Id$ + +TOP= ../../.. + +LIB= test + +SRCS= test.c \ + test_runner.c + +INCS= test.h + +WARNS?= 6 + +MAN= test.3 + +.include "${TOP}/mk/elftoolchain.lib.mk" diff --git a/test/libtest/lib/test.3 b/test/libtest/lib/test.3 new file mode 100644 index 0000000000000..c7045b4039c4f --- /dev/null +++ b/test/libtest/lib/test.3 @@ -0,0 +1,110 @@ +.\" Copyright (c) 2018, Joseph Koshy. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" This software is provided by Joseph Koshy ``as is'' and +.\" any express or implied warranties, including, but not limited to, the +.\" implied warranties of merchantability and fitness for a particular purpose +.\" are disclaimed. in no event shall Joseph Koshy be liable +.\" for any direct, indirect, incidental, special, exemplary, or consequential +.\" damages (including, but not limited to, procurement of substitute goods +.\" or services; loss of use, data, or profits; or business interruption) +.\" however caused and on any theory of liability, whether in contract, strict +.\" liability, or tort (including negligence or otherwise) arising in any way +.\" out of the use of this software, even if advised of the possibility of +.\" such damage. +.\" +.\" $Id$ +.\" +.Dd December 25, 2018 +.Dt TEST 3 +.Os +.Sh NAME +.Nm test +.Nd API for writing tests +.Sh LIBRARY +.Lb libtest +.Sh SYNOPSIS +.In test.h +.Ft enum testcase_status +.Fn testcase_setup "testcase_state *state" +.Ft enum testcase_status +.Fn testcase_teardown "testcase_state state" +.Ft enum test_result +.Fn test_function "testcase_state state" +.Vt "const char" +.Va test_description [] ; +.Vt "const char *" +.Va test_tags [] ; +.Vt "const char" +.Va testcase_description [] ; +.Vt "const char *" +.Va testcase_tags [] ; +.Sh DESCRIPTION +The +.Lb libtest +implements an API for writing tests. +.Ss Concepts +Tests are implemented using test functions, where each test function +verifies a specific assertion about the system being tested. +Test functions are associated with the following: +.Bl -bullet -compact +.It +An optional human-readable test description. +.It +An optional set of tags. +Tags are used to select or deselect specific tests in a run. +.El +.Pp +Test functions are further grouped into test cases, where a test case +contains a logical group of assertions about the system under test. +Test cases are associated the following: +.Bl -bullet -compact +.It +An optional human-readable test case description. +.It +An optional test case set up function. +If specified, this set up function would be invoked prior to any test +function contained in the test case. +The set up function can allocate and initialize test-specific state, to be +passed to test functions. +If no set up function is specified for the test case, a default no-op +function will be supplied. +.It +An optional test case tear down function. +The tear down function will be invoked after all test functions have been +invoked. +It would be responsible for deallocating any resources that its corresponding +set up function had allocated. +If no tear down function is specified for a test case, a default no-op +function will be supplied. +.It +An optional set of tags for the test case. +These tags are used to select or delect specific test cases in a given test +run. +.El +.Pp +One or more test cases would be linked with a test driver to form a +test executable. +The default test driver supplied allows the test cases and specific tests +within the executable to be specified on the command line. +.Ss Scaffolding Generation +The +.Xr make-test-scaffolding 1 +script will generate the scaffolding needed to produce a test executable +from object files containing symbols following its naming conventions. +.Sh SEE ALSO +.Xr make-test-scaffolding 1 +.Sh AUTHORS +The +.Lb libtest +was written by +.An Joseph Koshy Aq Mt jkoshy@users.sourceforge.net . diff --git a/test/libtest/lib/test.c b/test/libtest/lib/test.c new file mode 100644 index 0000000000000..04999c1966acf --- /dev/null +++ b/test/libtest/lib/test.c @@ -0,0 +1,29 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "test.h" + +/* To be implemented. */ diff --git a/test/libtest/lib/test.h b/test/libtest/lib/test.h new file mode 100644 index 0000000000000..86f91463993b6 --- /dev/null +++ b/test/libtest/lib/test.h @@ -0,0 +1,159 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIBTEST_TEST_H_ +#define _LIBTEST_TEST_H_ + +/* + * The return values from test functions. + * + * - TEST_PASS : The assertion(s) in the test function passed. + * - TEST_FAIL : At least one assertion in the test function failed. + * - TEST_UNRESOLVED : The assertions in the test function could not be + * checked for some reason. + */ +enum test_result { + TEST_PASS = 0, + TEST_FAIL = 1, + TEST_UNRESOLVED = 2 +}; + +/* + * The return values from test case set up and tear down functions. + * + * - TESTCASE_OK : The set up or tear down function was successful. + * - TESTCASE_ERROR : Set up or tear down actions could not be completed. + * + * If a test case set up function returns TESTCASE_ERROR then: + * - The test functions in the test case will not be run. + * - The test case's tear down function will not be invoked. + * - The test run as a whole will be treated as being in error. + * + * If a test case tear down function returns a TESTCASE_ERROR, then + * the test run as a whole be treated as being in error. + */ +enum testcase_status { + TESTCASE_OK = 0, + TESTCASE_ERROR = 1 +}; + +/* + * A testcase_state denotes resources that are shared by the test + * functions that are part of a test case. A testcase_state is allocated + * by the set up function for a test case. Conversely the test case's + * tear down function is responsible for deallocating the resources + * allocated by the set up function. + * + * The test(3) framework treats a testcase_state as an opaque value. + */ +typedef void *testcase_state; + +/* + * Test case and test function descriptions, and convenience macros + * to define these. + */ +typedef const char testcase_description[]; + +#if !defined(TEST_DESCRIPTION) +#define TEST_DESCRIPTION(NAME) test_description tf_description_##NAME +#endif + +typedef const char test_description[]; + +#if !defined(TESTCASE_DESCRIPTION) +#define TESTCASE_DESCRIPTION(NAME) testcase_description tc_description_##NAME +#endif + +/* + * Test case and test function tags, and convenience macros to define + * these. + */ +typedef const char *testcase_tags[]; + +#if !defined(TESTCASE_TAGS) +#define TESTCASE_TAGS(NAME) testcase_tags tc_tags_##NAME +#endif + +typedef const char *test_tags[]; +#if !defined(TEST_TAGS) +#define TEST_TAGS(NAME) test_tags tf_tags_##NAME +#endif + +/* + * A test case set up function. + * + * If defined for a test case, this function will be called prior to + * the execution of an of the test functions within the test cae. Test + * case execution will be aborted if the function returns any value other + * than TESTCASE_OK. + * + * The function can set '*state' to a memory area holding test state to be + * passed to test functions. + * + * If the test case does not define a set up function, then a default + * no-op set up function will be used. + */ +typedef enum testcase_status (test_case_setup_function) + (testcase_state *state); + +/* + * A test function. + * + * This function will be invoked with the state that had been set by the + * test case set up function. The function returns TEST_PASS to report that + * its test succeeded or TEST_FAIL otherwise. In the event the test could + * not be executed, it can return TEST_UNRESOLVED. + */ +typedef enum test_result (test_function)(testcase_state state); + +/* + * A test case tear down function. + * + * If defined for a test case, this function will be called after the + * execution of the test functions in the test case. It is passed the + * state that had been allocated by the test case set up function, and is + * responsible for deallocating the resources that the set up function + * had allocated. + */ +typedef enum testcase_status (test_case_teardown_function)(testcase_state state); + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Write a progress report to the test log. + * + * This function takes a printf(3)-like format string and associated + * arguments. + */ +int test_report_progress(const char *format, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* _LIBTEST_TEST_H_ */ diff --git a/test/libtest/lib/test_runner.c b/test/libtest/lib/test_runner.c new file mode 100644 index 0000000000000..1366ff4a538e6 --- /dev/null +++ b/test/libtest/lib/test_runner.c @@ -0,0 +1,31 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * An implementation of a test driver for test(3) tests. + */ + +/* To be implemented. */ diff --git a/test/libtest/lib/test_runner.h b/test/libtest/lib/test_runner.h new file mode 100644 index 0000000000000..cbf00f29b44da --- /dev/null +++ b/test/libtest/lib/test_runner.h @@ -0,0 +1,118 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIBTEST_TEST_RUNNER_H_ +#define _LIBTEST_TEST_RUNNER_H_ + +#include "test.h" + +/* + * These data structures and functions are used by test driver that + * execute tests. + */ + +/* + * The completion status for a test run: + * + * - TESTRUN_PASS : All test cases were successfully invoked and all test + * purposes in the test cases passed. + * - TESTRUN_FAIL : All test cases were successfully invoked but at least + * one test purpose reported a test failure. + * - TESTRUN_ERROR : At least one test case reported an error during its + * set up or tear down phase. + */ +enum testrun_status { + TESTRUN_PASS = 0, + TESTRUN_FAIL = 1, + TESTRUN_ERROR = 2 +}; + +/* + * A single test function, with its associated tags and description. + */ +struct test_descriptor { + const char *t_name; /* Test name. */ + const char *t_description; /* Test description. */ + const char **t_tags; /* Tags associated with the test. */ + test_function *t_func; /* The function to invoke. */ +}; + +/* + * A test case. + */ +struct test_case_descriptor { + const char *tc_name; /* Test case name. */ + const char *tc_description; /* Test case description. */ + const char **tc_tags; /* Any associated tags. */ + struct test_descriptor *tc_tests; /* The tests in this test case. */ +}; + +/* + * All test cases. + */ +extern struct test_case_descriptor test_cases[]; + +enum testrun_style { + /* Libtest semantics. */ + TESTRUN_STYLE_LIBTEST, + + /* + * Be compatible with the Test Anything Protocol + * (http://testanything.org/). + */ + TESTRUN_STYLE_TAP, + + /* Be compatible with NetBSD ATF(9). */ + TESTRUN_STYLE_ATF +}; + +/* + * Parameters for the run. + */ +struct test_run { + /* + * An optional name assigned by the user for this test run. + * + * This name is reported in test logs and is not interpreted + * by the test harness. + */ + char *testrun_name; + + /* The source directory for the run. */ + char *testrun_source_directory; + + /* The directory in which the test is executing. */ + char *testrun_test_directory; +}; + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif + +#endif /* _LIBTEST_TEST_RUNNER_H_ */ |