diff options
Diffstat (limited to 'test/libtest')
| -rw-r--r-- | test/libtest/Makefile | 5 | ||||
| -rw-r--r-- | test/libtest/README.rst | 6 | ||||
| -rwxr-xr-x | test/libtest/bin/make-test-scaffolding | 29 | ||||
| -rw-r--r-- | test/libtest/driver/Makefile | 7 | ||||
| -rw-r--r-- | test/libtest/driver/driver.c | 216 | ||||
| -rw-r--r-- | test/libtest/driver/driver.h | 206 | ||||
| -rw-r--r-- | test/libtest/driver/driver_main.c | 726 | ||||
| -rw-r--r-- | test/libtest/driver/test_driver.1 | 308 | ||||
| -rw-r--r-- | test/libtest/examples/minimal_example.c | 8 | ||||
| -rw-r--r-- | test/libtest/examples/simple_example.c | 48 | ||||
| -rw-r--r-- | test/libtest/lib/Makefile | 5 | ||||
| -rw-r--r-- | test/libtest/lib/test.3 | 20 | ||||
| -rw-r--r-- | test/libtest/lib/test.h | 54 | ||||
| -rw-r--r-- | test/libtest/lib/test_case.h (renamed from test/libtest/driver/test_main.c) | 43 | ||||
| -rw-r--r-- | test/libtest/lib/test_runner.c | 31 | ||||
| -rw-r--r-- | test/libtest/lib/test_runner.h | 118 |
16 files changed, 1594 insertions, 236 deletions
diff --git a/test/libtest/Makefile b/test/libtest/Makefile index 1dc489b9db8a..3cf6de08f316 100644 --- a/test/libtest/Makefile +++ b/test/libtest/Makefile @@ -9,9 +9,8 @@ SUBDIR+= lib SUBDIR+= driver SUBDIR+= examples -.if !make(install) +.if !make(install) && !make(test) .include "$(TOP)/mk/elftoolchain.subdir.mk" .else -install: .SILENT .PHONY - echo Nothing to install. +install test: .SILENT .PHONY .endif diff --git a/test/libtest/README.rst b/test/libtest/README.rst index 3f29c85f8a93..b93dca7a69ee 100644 --- a/test/libtest/README.rst +++ b/test/libtest/README.rst @@ -43,7 +43,7 @@ functions contained in a test case named "``helloworld``": /* File: test.c */ #include "test.h" - TESTCASE_DESCRIPTION(helloworld) = + TEST_CASE_DESCRIPTION(helloworld) = "A description of the helloworld test case."; enum test_result @@ -69,14 +69,14 @@ Test cases can define their own set up and tear down functions: tc_setup_helloworld(testcase_state *tcs) { *tcs = ..allocate a struct helloworld_test.. ; - return (TESTCASE_OK); + return (TEST_CASE_OK); } enum testcase_status tc_teardown_helloworld(testcase_state tcs) { .. deallocate test case state.. - return (TESTCASE_OK); + return (TEST_CASE_OK); } The set up function for a test case will be invoked prior to any of diff --git a/test/libtest/bin/make-test-scaffolding b/test/libtest/bin/make-test-scaffolding index c0966d34527b..6ec78e8224e1 100755 --- a/test/libtest/bin/make-test-scaffolding +++ b/test/libtest/bin/make-test-scaffolding @@ -79,7 +79,7 @@ cat <<EOF /* GENERATED FROM: ${@} */ #include <stddef.h> #include "test.h" -#include "test_runner.h" +#include "test_case.h" EOF if ! nm ${*} | sort -k 3 | \ @@ -108,9 +108,13 @@ if ! nm ${*} | sort -k 3 | \ function print_test_case_record(tc_name) { printf("\t{\n") printf("\t\t.tc_name = \"%s\",\n", tc_name) - printf("\t\t.tc_description = %s,\n", test_case_descriptions[tc_name]) + printf("\t\t.tc_description = %s,\n", + test_case_descriptions[tc_name]) printf("\t\t.tc_tags = %s,\n", test_case_tags[tc_name]) - printf("\t\t.tc_tests = test_functions_%s\n", tc_name) + tf_name = "test_functions_" tc_name + printf("\t\t.tc_tests = %s,\n", tf_name) + printf("\t\t.tc_count = sizeof (%s) / sizeof (%s[0]),\n", + tf_name, tf_name) printf("\t},\n") } function delete_test_functions(tc_name) { @@ -120,16 +124,19 @@ if ! nm ${*} | sort -k 3 | \ } } function print_test_functions_record(tc_name) { - printf("struct test_descriptor test_functions_%s[] = {\n", tc_name) + printf("struct test_function_descriptor test_functions_%s[]", + tc_name) + printf(" = {\n") for (tf_name in test_functions) { if (tc_name != matched_test_case(tf_name)) continue printf("\t{\n") - printf("\t\t.t_name = \"%s\",\n", tf_name) - printf("\t\t.t_description = %s,\n", + printf("\t\t.tf_name = \"%s\",\n", tf_name) + printf("\t\t.tf_description = %s,\n", test_function_descriptions[tf_name]) - printf("\t\t.t_func = %s,\n", prefix_tf tf_name) - printf("\t\t.t_tags = %s\n", test_function_tags[tf_name]) + printf("\t\t.tf_func = %s,\n", prefix_tf tf_name) + printf("\t\t.tf_tags = %s\n", + test_function_tags[tf_name]) printf("\t},\n") } printf("};\n") @@ -144,7 +151,7 @@ if ! nm ${*} | sort -k 3 | \ test_case_tags[DEFAULT] = "NULL" } ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tc_descr { - printf("extern testcase_description %s;\n", $3) + printf("extern test_case_description %s;\n", $3) tc_name = suffix($3, prefix_tc_descr) test_cases[tc_name] = 1 test_case_descriptions[tc_name] = $3 @@ -155,7 +162,7 @@ if ! nm ${*} | sort -k 3 | \ test_case_setup[tc_name] = $3 } ($2 == "R" || $2 == "D") && $3 ~ "^" prefix_tc_tags { - printf("extern testcase_tags %s;\n", $3) + printf("extern test_case_tags %s;\n", $3) tc_name = suffix($3, prefix_tc_tags) test_cases[tc_name] = 1 test_case_tags[tc_name] = $3 @@ -206,6 +213,8 @@ if ! nm ${*} | sort -k 3 | \ if (needs_default) print_test_case_record(DEFAULT) printf("};\n") + printf("const int test_case_count = sizeof(test_cases) / ") + printf("sizeof(test_cases[0]);\n") }'; then # Cleanup in case of an error. rm ${output_file} diff --git a/test/libtest/driver/Makefile b/test/libtest/driver/Makefile index d149fee159d4..87788d3fb13c 100644 --- a/test/libtest/driver/Makefile +++ b/test/libtest/driver/Makefile @@ -6,9 +6,12 @@ TOP= ../../.. CFLAGS+= -I${TOP}/test/libtest/lib -LIB= test_main -SRCS= test_main.c +LIB= driver +SRCS= driver.c \ + driver_main.c WARNS?= 6 +MAN= test_driver.1 + .include "$(TOP)/mk/elftoolchain.lib.mk" diff --git a/test/libtest/driver/driver.c b/test/libtest/driver/driver.c new file mode 100644 index 000000000000..aa85c7cb4ddf --- /dev/null +++ b/test/libtest/driver/driver.c @@ -0,0 +1,216 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * The implementation of the test driver. + */ + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/time.h> + +#include <err.h> +#include <libgen.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sysexits.h> + +#include "driver.h" + +#if defined(ELFTC_VCSID) +ELFTC_VCSID("$Id$"); +#endif + +#define SYSTEM_TMPDIR_ENV_VAR "TMPDIR" + +bool +test_driver_add_search_path(struct test_run *tr, const char *directory_name) +{ + char *canonical_path; + struct test_search_path_entry *entry; + + if (!test_driver_is_directory(directory_name)) + return (false); + + if ((canonical_path = realpath(directory_name, NULL)) == NULL) + err(1, "Cannot determine the canonical path for \"%s\"", + directory_name); + + /* Look for, and ignore duplicates. */ + STAILQ_FOREACH(entry, &tr->tr_search_path, tsp_next) { + if (strcmp(canonical_path, entry->tsp_directory) == 0) + return (true); + } + + entry = calloc(1, sizeof(*entry)); + entry->tsp_directory = canonical_path; + + STAILQ_INSERT_TAIL(&tr->tr_search_path, entry, tsp_next); + + return (true); +} + +/* + * Return an initialized test run descriptor. + * + * The caller should use test_driver_free_run() to release the returned + * descriptor. + */ +struct test_run * +test_driver_allocate_run(void) +{ + struct test_run *tr; + + tr = calloc(sizeof(struct test_run), 1); + tr->tr_action = TEST_RUN_EXECUTE; + tr->tr_style = TR_STYLE_LIBTEST; + STAILQ_INIT(&tr->tr_test_cases); + STAILQ_INIT(&tr->tr_search_path); + + return (tr); +} + +/* + * Destroy an allocated test run descriptor. + * + * The passed in pointer should not be used after this function returns. + */ +void +test_driver_free_run(struct test_run *tr) +{ + struct test_search_path_entry *path_entry; + struct test_case_selector *test_case_entry; + struct test_function_selector *function_entry; + + free(tr->tr_runtime_base_directory); + free(tr->tr_name); + if (tr->tr_artefact_archive) + free(tr->tr_artefact_archive); + + /* Free the search path list. */ + while (!STAILQ_EMPTY(&tr->tr_search_path)) { + path_entry = STAILQ_FIRST(&tr->tr_search_path); + STAILQ_REMOVE_HEAD(&tr->tr_search_path, tsp_next); + free(path_entry); + } + + /* Free the test selector list. */ + while (!STAILQ_EMPTY(&tr->tr_test_cases)) { + test_case_entry = STAILQ_FIRST(&tr->tr_test_cases); + STAILQ_REMOVE_HEAD(&tr->tr_test_cases, tcs_next); + + /* Free the linked test functions. */ + while (!STAILQ_EMPTY(&test_case_entry->tcs_functions)) { + function_entry = + STAILQ_FIRST(&test_case_entry->tcs_functions); + STAILQ_REMOVE_HEAD(&test_case_entry->tcs_functions, + tfs_next); + + free(function_entry); + } + + free(test_case_entry); + } + + free(tr); +} + +/* + * Populate unset fields of a struct test_run with defaults. + */ +bool +test_driver_finish_run_initialization(struct test_run *tr, const char *argv0) +{ + struct timeval tv; + const char *basedir; + const char *search_path; + const char *last_component; + char *argv0_copy, *path_copy, *path_element; + char test_name[NAME_MAX]; + + if (tr->tr_name == NULL) { + /* Per POSIX, basename(3) can modify its argument. */ + argv0_copy = strdup(argv0); + last_component = basename(argv0_copy); + + if (gettimeofday(&tv, NULL)) + return (false); + + (void) snprintf(test_name, sizeof(test_name), "%s+%ld%ld", + last_component, (long) tv.tv_sec, (long) tv.tv_usec); + + tr->tr_name = strdup(test_name); + + free(argv0_copy); + } + + /* + * Select a base directory, if one was not specified. + */ + if (tr->tr_runtime_base_directory == NULL) { + basedir = getenv(TEST_TMPDIR_ENV_VAR); + if (basedir == NULL) + basedir = getenv(SYSTEM_TMPDIR_ENV_VAR); + if (basedir == NULL) + basedir = "/tmp"; + tr->tr_runtime_base_directory = realpath(basedir, NULL); + if (tr->tr_runtime_base_directory == NULL) + err(1, "realpath(%s) failed", basedir); + } + + /* + * Add the search paths specified by the environment variable + * 'TEST_PATH' to the end of the search list. + */ + if ((search_path = getenv(TEST_SEARCH_PATH_ENV_VAR)) != NULL && + *search_path != '\0') { + path_copy = strdup(search_path); + path_element = strtok(path_copy, ":"); + do { + if (!test_driver_add_search_path(tr, path_element)) + warnx("in environment variable \"%s\": path " + "\"%s\" does not name a directory.", + TEST_SEARCH_PATH_ENV_VAR, path_element); + } while ((path_element = strtok(NULL, ":")) != NULL); + } + + return (true); +} + +/* + * Helper: return true if the passed in path names a directory, or false + * otherwise. + */ +bool +test_driver_is_directory(const char *path) +{ + struct stat sb; + if (stat(path, &sb) != 0) + return false; + return S_ISDIR(sb.st_mode); +} diff --git a/test/libtest/driver/driver.h b/test/libtest/driver/driver.h new file mode 100644 index 000000000000..5ae5cfd81246 --- /dev/null +++ b/test/libtest/driver/driver.h @@ -0,0 +1,206 @@ +/*- + * Copyright (c) 2018,2019 Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIBTEST_DRIVER_H_ +#define _LIBTEST_DRIVER_H_ + +#include <sys/queue.h> + +#include <limits.h> +#include <stdbool.h> + +#include "_elftc.h" + +#include "test.h" + +#define TEST_SEARCH_PATH_ENV_VAR "TEST_PATH" +#define TEST_TMPDIR_ENV_VAR "TEST_TMPDIR" + +/* + * Run time data strucrures. + */ + +/* The completion status for a test run */ +enum test_run_status { + /* + * All test cases were successfully invoked, and all their contained + * test purposes passed. + */ + TR_STATUS_PASS = 0, + + /* + * All test cases were successfully invoked but at least one test + * function reported a failure. + */ + TR_STATUS_FAIL = 1, + + /* + * At least one test case reported an error during its set up or tear + * down phase. + */ + TR_STATUS_ERROR = 2 +}; + +/* + * The 'style' of the run determines the manner in which the test + * executable reports test status. + */ +enum test_run_style { + /* Libtest semantics. */ + TR_STYLE_LIBTEST, + + /* + * Be compatible with the Test Anything Protocol + * (http://testanything.org/). + */ + TR_STYLE_TAP, + + /* Be compatible with NetBSD ATF(9). */ + TR_STYLE_ATF +}; + +/* + * Structures used for selecting tests. + */ +struct test_function_selector { + const struct test_function_descriptor *tfs_descriptor; + + STAILQ_ENTRY(test_function_selector) tfs_next; + int tfs_is_selected; +}; + +STAILQ_HEAD(test_function_selector_list, test_function_selector); + +struct test_case_selector { + const struct test_case_descriptor *tcs_descriptor; + STAILQ_ENTRY(test_case_selector) tcs_next; + struct test_function_selector_list tcs_functions; + int tcs_selected_count; +}; + +/* + * The action being requested of the test driver. + */ +enum test_run_action { + TEST_RUN_EXECUTE, /* Execute the selected tests. */ + TEST_RUN_LIST, /* Only list tests. */ +}; + +STAILQ_HEAD(test_case_selector_list, test_case_selector); + +/* + * Runtime directories to look up data files. + */ +struct test_search_path_entry { + char *tsp_directory; + STAILQ_ENTRY(test_search_path_entry) tsp_next; +}; + +STAILQ_HEAD(test_search_path_list, test_search_path_entry); + +/* + * Used to track flags that were explicity set on the command line. + */ +enum test_run_flags { + TRF_BASE_DIRECTORY = 1U << 0, + TRF_EXECUTION_TIME = 1U << 1, + TRF_ARTEFACT_ARCHIVE = 1U << 2, + TRF_NAME = 1U << 3, + TRF_SEARCH_PATH = 1U << 4, + TRF_EXECUTION_STYLE = 1U << 5, +}; + +/* + * Parameters for the run. + */ +struct test_run { + /* + * Flags tracking the options which were explicitly set. + * + * This field is a bitmask formed of 'enum test_run_flags' values. + */ + unsigned int tr_commandline_flags; + + /* What the test run should do. */ + enum test_run_action tr_action; + + /* The desired behavior of the test harness. */ + enum test_run_style tr_style; + + /* The desired verbosity level. */ + int tr_verbosity; + + /* An optional name assigned by the user for this test run. */ + char *tr_name; + + /* + * The absolute path to the directory under which the test is + * to be run. + * + * Each test case will be invoked in some subdirectory of this + * directory. + */ + char *tr_runtime_base_directory; + + /* + * The test timeout in seconds. + * + * A value of zero indicates that the test driver should wait + * indefinitely for tests. + */ + long tr_max_seconds_per_test; + + /* + * If not NULL, An absolute pathname to an archive that will hold + * the artefacts created by a test run. + */ + char *tr_artefact_archive; + + /* + * Directories to use when resolving non-absolute data file + * names. + */ + struct test_search_path_list tr_search_path; + + /* All tests selected for this run. */ + struct test_case_selector_list tr_test_cases; +}; + +#ifdef __cplusplus +extern "C" { +#endif +struct test_run *test_driver_allocate_run(void); +bool test_driver_add_search_path(struct test_run *, + const char *search_path); +void test_driver_free_run(struct test_run *); +bool test_driver_is_directory(const char *); +bool test_driver_finish_run_initialization(struct test_run *, + const char *argv0); +#ifdef __cplusplus +} +#endif + +#endif /* _LIBTEST_DRIVER_H_ */ diff --git a/test/libtest/driver/driver_main.c b/test/libtest/driver/driver_main.c new file mode 100644 index 000000000000..c43ccf52ca15 --- /dev/null +++ b/test/libtest/driver/driver_main.c @@ -0,0 +1,726 @@ +/*- + * Copyright (c) 2018, Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This file defines a "main()" that invokes (or lists) the tests that were + * linked into the current executable. + */ + +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/stat.h> + +#include <assert.h> +#include <err.h> +#include <errno.h> +#include <fnmatch.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sysexits.h> +#include <time.h> +#include <unistd.h> + +#include "_elftc.h" + +#include "test.h" +#include "test_case.h" + +#include "driver.h" + +#if defined(ELFTC_VCSID) +ELFTC_VCSID("$Id$"); +#endif + +enum selection_scope { + SCOPE_TEST_CASE = 0, /* c:STRING */ + SCOPE_TEST_FUNCTION, /* f:STRING */ + SCOPE_TAG, /* t:STRING */ +}; + +/* Selection list entry. */ +struct selection_option { + STAILQ_ENTRY(selection_option) so_next; + + /* The text to use for matching. */ + const char *so_pattern; + + /* + * Whether matched test and test cases should be selected + * (if false) or deselected (if true). + */ + bool so_select_tests; + + /* The kind of information to match. */ + enum selection_scope so_selection_scope; +}; + +/* All selection options specified. */ +STAILQ_HEAD(selection_option_list, selection_option); + +static struct selection_option * +parse_selection_option(const char *option) +{ + int scope_char; + bool select_tests; + enum selection_scope scope; + struct selection_option *so; + + scope_char = '\0'; + select_tests = true; + scope = SCOPE_TEST_CASE; + + /* Deselection patterns start with a '-'. */ + if (*option == '-') { + select_tests = false; + option++; + } + + /* + * If a scope was not specified, the selection scope defaults + * to SCOPE_TEST_CASE. + */ + if (strchr(option, ':') == NULL) + scope_char = 'c'; + else { + scope_char = *option++; + if (*option != ':') + return (NULL); + option++; /* Skip over the ':'. */ + } + + if (*option == '\0') + return (NULL); + + switch (scope_char) { + case 'c': + scope = SCOPE_TEST_CASE; + break; + case 'f': + scope = SCOPE_TEST_FUNCTION; + break; + case 't': + scope = SCOPE_TAG; + break; + default: + return (NULL); + } + + so = calloc(1, sizeof(*so)); + so->so_pattern = option; + so->so_selection_scope = scope; + so->so_select_tests = select_tests; + + return (so); +} + +/* Test execution styles. */ +struct style_entry { + enum test_run_style se_style; + const char *se_name; +}; + +static const struct style_entry known_styles[] = { + { TR_STYLE_LIBTEST, "libtest" }, + { TR_STYLE_TAP, "tap" }, + { TR_STYLE_ATF, "atf" } +}; + +/* + * Parse a test run style. + * + * This function returns true if the run style was recognized, or + * false otherwise. + */ +static bool +parse_run_style(const char *option, enum test_run_style *run_style) +{ + size_t n; + + for (n = 0; n < sizeof(known_styles) / sizeof(known_styles[0]); n++) { + if (strcasecmp(option, known_styles[n].se_name) == 0) { + *run_style = known_styles[n].se_style; + return (true); + } + } + + return (false); +} + +/* + * Return the canonical spelling of a test execution style. + */ +static const char * +to_execution_style_name(enum test_run_style run_style) +{ + size_t n; + + for (n = 0; n < sizeof(known_styles) / sizeof(known_styles[0]); n++) { + if (known_styles[n].se_style == run_style) + return (known_styles[n].se_name); + } + + return (NULL); +} + +/* + * Parse a string value containing a positive integral number. + */ +static bool +parse_execution_time(const char *option, long *execution_time) { + char *end; + long value; + + if (option == NULL || *option == '\0') + return (false); + + value = strtol(option, &end, 10); + + /* Check for parse errors. */ + if (*end != '\0') + return (false); + + /* Reject negative numbers. */ + if (value < 0) + return (false); + + /* Check for overflows during parsing. */ + if (value == LONG_MAX && errno == ERANGE) + return (false); + + *execution_time = value; + + return (true); +} + +/* + * Match the names of test cases. + * + * In the event of a match, then the selection state specifed in + * 'option' is applied to all the test functions in the test case. + */ +static void +match_test_cases(struct selection_option *option, + struct test_case_selector *tcs) +{ + const struct test_case_descriptor *tcd; + struct test_function_selector *tfs; + + tcd = tcs->tcs_descriptor; + + if (fnmatch(option->so_pattern, tcd->tc_name, 0)) + return; + + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) + tfs->tfs_is_selected = option->so_select_tests; +} + +/* + * Match the names of test functions. + */ +static void +match_test_functions(struct selection_option *option, + struct test_case_selector *tcs) +{ + struct test_function_selector *tfs; + const struct test_function_descriptor *tfd; + + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) { + tfd = tfs->tfs_descriptor; + + if (fnmatch(option->so_pattern, tfd->tf_name, 0)) + continue; + + tfs->tfs_is_selected = option->so_select_tests; + } +} + +/* + * Helper: returns true if the specified text matches any of the + * entries in the array 'tags'. + */ +static bool +match_tags_helper(const char *pattern, const char *tags[]) +{ + const char **tag; + + if (!tags) + return (false); + + for (tag = tags; *tag && **tag != '\0'; tag++) { + if (!fnmatch(pattern, *tag, 0)) + return (true); + } + + return (false); +} + +/* + * Match tags. + * + * Matches against test case tags apply to all the test + * functions in the test case. + * + * Matches against test function tags apply to the matched + * test function only. + */ +static void +match_tags(struct selection_option *option, + struct test_case_selector *tcs) +{ + const struct test_case_descriptor *tcd; + const struct test_function_descriptor *tfd; + struct test_function_selector *tfs; + + tcd = tcs->tcs_descriptor; + + /* + * If the tag in the option matches a tag associated with + * a test case, then we set all of the test case's functions + * to the specified selection state. + */ + if (match_tags_helper(option->so_pattern, tcd->tc_tags)) { + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) + tfs->tfs_is_selected = option->so_select_tests; + return; + } + + /* + * Otherwise, check the tag against the tags for each function + * in the test case and set the selection state of each matched + * function. + */ + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) { + tfd = tfs->tfs_descriptor; + if (match_tags_helper(option->so_pattern, tfd->tf_tags)) + tfs->tfs_is_selected = option->so_select_tests; + } +} + +/* + * Add the selected tests to the test run. + * + * The memory used by the options list is returned to the system when this + * function completes. + */ +static void +select_tests(struct test_run *tr, + struct selection_option_list *selections) +{ + int i, j; + struct selection_option *selection; + const struct test_case_descriptor *tcd; + struct test_case_selector *tcs; + struct test_function_selector *tfs; + bool default_selection_state; + int selected_count; + + default_selection_state = STAILQ_EMPTY(selections); + + /* + * Set up runtime descriptors. + */ + for (i = 0; i < test_case_count; i++) { + if ((tcs = calloc(1, sizeof(*tcs))) == NULL) + err(EX_OSERR, "cannot allocate a test-case selector"); + STAILQ_INSERT_TAIL(&tr->tr_test_cases, tcs, tcs_next); + STAILQ_INIT(&tcs->tcs_functions); + + tcd = &test_cases[i]; + + tcs->tcs_descriptor = tcd; + + for (j = 0; j < tcd->tc_count; j++) { + if ((tfs = calloc(1, sizeof(*tfs))) == NULL) + err(EX_OSERR, "cannot allocate a test " + "function selector"); + STAILQ_INSERT_TAIL(&tcs->tcs_functions, tfs, tfs_next); + + tfs->tfs_descriptor = tcd->tc_tests + j; + tfs->tfs_is_selected = default_selection_state; + } + } + + /* + * Set or reset the selection state based on the options. + */ + STAILQ_FOREACH(selection, selections, so_next) { + STAILQ_FOREACH(tcs, &tr->tr_test_cases, tcs_next) { + switch (selection->so_selection_scope) { + case SCOPE_TEST_CASE: + match_test_cases(selection, tcs); + break; + case SCOPE_TEST_FUNCTION: + match_test_functions(selection, tcs); + break; + case SCOPE_TAG: + match_tags(selection, tcs); + break; + } + } + } + + /* + * Determine the count of tests selected, for each test case. + */ + STAILQ_FOREACH(tcs, &tr->tr_test_cases, tcs_next) { + selected_count = 0; + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) + selected_count += tfs->tfs_is_selected; + tcs->tcs_selected_count = selected_count; + } + + /* Free up the selection list. */ + while (!STAILQ_EMPTY(selections)) { + selection = STAILQ_FIRST(selections); + STAILQ_REMOVE_HEAD(selections, so_next); + free(selection); + } +} + +/* + * Translate a file name to absolute form. + * + * The caller needs to free the returned pointer. + */ +static char * +to_absolute_path(const char *filename) +{ + size_t space_needed; + char *absolute_path; + char current_directory[PATH_MAX]; + + if (filename == NULL || *filename == '\0') + return (NULL); + if (*filename == '/') + return strdup(filename); + + if (getcwd(current_directory, sizeof(current_directory)) == NULL) + err(1, "getcwd failed"); + + /* Reserve space for the slash separator and the trailing NUL. */ + space_needed = strlen(current_directory) + strlen(filename) + 2; + if ((absolute_path = malloc(space_needed)) == NULL) + err(1, "malloc failed"); + if (snprintf(absolute_path, space_needed, "%s/%s", current_directory, + filename) != (int) (space_needed - 1)) + err(1, "snprintf failed"); + return (absolute_path); +} + + +/* + * Display run parameters. + */ + +#define FIELD_NAME_WIDTH 24 +#define INFOLINE(NAME, FLAG, FORMAT, ...) do { \ + printf("I %c %-*s " FORMAT, \ + (FLAG) ? '!' : '.', \ + FIELD_NAME_WIDTH, NAME, __VA_ARGS__); \ + } while (0) + +static void +show_run_header(const struct test_run *tr) +{ + time_t start_time; + struct test_search_path_entry *path_entry; + + if (tr->tr_verbosity == 0) + return; + + INFOLINE("test-run-name", tr->tr_commandline_flags & TRF_NAME, + "%s\n", tr->tr_name); + + INFOLINE("test-execution-style", + tr->tr_commandline_flags & TRF_EXECUTION_STYLE, + "%s\n", to_execution_style_name(tr->tr_style)); + + if (!STAILQ_EMPTY(&tr->tr_search_path)) { + INFOLINE("test-search-path", + tr->tr_commandline_flags & TRF_SEARCH_PATH, + "%c", '['); + STAILQ_FOREACH(path_entry, &tr->tr_search_path, tsp_next) { + printf(" %s", path_entry->tsp_directory); + } + printf(" ]\n"); + } + + INFOLINE("test-run-base-directory", + tr->tr_commandline_flags & TRF_BASE_DIRECTORY, + "%s\n", tr->tr_runtime_base_directory); + + if (tr->tr_artefact_archive) { + INFOLINE("test-artefact-archive", + tr->tr_commandline_flags & TRF_ARTEFACT_ARCHIVE, + "%s\n", tr->tr_artefact_archive); + } + + printf("I %c %-*s ", + tr->tr_commandline_flags & TRF_EXECUTION_TIME ? '=' : '.', + FIELD_NAME_WIDTH, "test-execution-time"); + if (tr->tr_max_seconds_per_test == 0) + printf("unlimited\n"); + else + printf("%lu\n", tr->tr_max_seconds_per_test); + + printf("I %% %-*s %d\n", FIELD_NAME_WIDTH, "test-case-count", + test_case_count); + + if (tr->tr_action == TEST_RUN_EXECUTE) { + start_time = time(NULL); + printf("I %% %-*s %s", FIELD_NAME_WIDTH, + "test-run-start-time", ctime(&start_time)); + } +} + +static void +show_run_trailer(const struct test_run *tr) +{ + time_t end_time; + + if (tr->tr_verbosity == 0) + return; + + if (tr->tr_action == TEST_RUN_EXECUTE) { + end_time = time(NULL); + printf("I %% %-*s %s", FIELD_NAME_WIDTH, "test-run-end-time", + asctime(localtime(&end_time))); + } +} + +#undef INFOLINE +#undef FIELD_HEADER_WIDTH + +/* + * Helper: returns a character indicating the selection status for + * a test case. This character is as follows: + * + * - "*" all test functions in the test case were selected. + * - "+" some test functions in the test case were selected. + * - "-" no test functions from the test case were selected. + */ +static int +get_test_case_status(const struct test_case_selector *tcs) +{ + if (tcs->tcs_selected_count == 0) + return '-'; + if (tcs->tcs_selected_count == tcs->tcs_descriptor->tc_count) + return '*'; + return '?'; +} + +/* + * Helper: print out a comma-separated list of tags. + */ +static void +show_tags(int indent, const char *tags[]) +{ + const char **tag; + + printf("%*c: ", indent, ' '); + for (tag = tags; *tag && **tag != '\0';) { + printf("%s", *tag++); + if (*tag && **tag != '\0') + printf(","); + } + printf("\n"); +} + +/* + * Display a test case descriptor. + */ +static void +show_test_case(struct test_run *tr, const struct test_case_selector *tcs) +{ + const struct test_case_descriptor *tcd; + int prefix_char; + + prefix_char = get_test_case_status(tcs); + tcd = tcs->tcs_descriptor; + + printf("C %c %s\n", prefix_char, tcd->tc_name); + + if (tr->tr_verbosity > 0 && tcd->tc_tags != NULL) + show_tags(2, tcd->tc_tags); + + if (tr->tr_verbosity > 1 && tcd->tc_description) + printf(" & %s\n", tcd->tc_description); +} + +static void +show_test_function(struct test_run *tr, + const struct test_function_selector *tfs) +{ + const struct test_function_descriptor *tfd; + int selection_char; + + selection_char = tfs->tfs_is_selected ? '*' : '-'; + tfd = tfs->tfs_descriptor; + + printf(" F %c %s\n", selection_char, tfd->tf_name); + + if (tr->tr_verbosity > 0 && tfd->tf_tags != NULL) + show_tags(4, tfd->tf_tags); + + if (tr->tr_verbosity > 1 && tfd->tf_description) + printf(" & %s\n", tfd->tf_description); +} + +static int +show_listing(struct test_run *tr) +{ + const struct test_case_selector *tcs; + const struct test_function_selector *tfs; + + STAILQ_FOREACH(tcs, &tr->tr_test_cases, tcs_next) { + show_test_case(tr, tcs); + STAILQ_FOREACH(tfs, &tcs->tcs_functions, tfs_next) + show_test_function(tr, tfs); + } + + return (EXIT_SUCCESS); +} + +int +main(int argc, char **argv) +{ + struct test_run *tr; + int exit_code, option; + enum test_run_style run_style; + struct selection_option *selector; + struct selection_option_list selections = + STAILQ_HEAD_INITIALIZER(selections); + + tr = test_driver_allocate_run(); + + /* Parse arguments. */ + while ((option = getopt(argc, argv, ":R:T:c:ln:p:s:t:v")) != -1) { + switch (option) { + case 'R': /* Test runtime directory. */ + if (!test_driver_is_directory(optarg)) + errx(EX_USAGE, "option -%c: argument \"%s\" " + "does not name a directory.", option, + optarg); + tr->tr_runtime_base_directory = realpath(optarg, NULL); + if (tr->tr_runtime_base_directory == NULL) + err(1, "realpath failed for \"%s\"", optarg); + tr->tr_commandline_flags |= TRF_BASE_DIRECTORY; + break; + case 'T': /* Max execution time for a test function. */ + if (!parse_execution_time( + optarg, &tr->tr_max_seconds_per_test)) + errx(EX_USAGE, "option -%c: argument \"%s\" " + "is not a valid execution time value.", + option, optarg); + tr->tr_commandline_flags |= TRF_EXECUTION_TIME; + break; + case 'c': /* The archive holding artefacts. */ + tr->tr_artefact_archive = to_absolute_path(optarg); + tr->tr_commandline_flags |= TRF_ARTEFACT_ARCHIVE; + break; + case 'l': /* List matching tests. */ + tr->tr_action = TEST_RUN_LIST; + break; + case 'n': /* Test run name. */ + if (tr->tr_name) + free(tr->tr_name); + tr->tr_name = strdup(optarg); + tr->tr_commandline_flags |= TRF_NAME; + break; + case 'p': /* Add a search path entry. */ + if (!test_driver_add_search_path(tr, optarg)) + errx(EX_USAGE, "option -%c: argument \"%s\" " + "does not name a directory.", option, + optarg); + tr->tr_commandline_flags |= TRF_SEARCH_PATH; + break; + case 's': /* Test execution style. */ + if (!parse_run_style(optarg, &run_style)) + errx(EX_USAGE, "option -%c: argument \"%s\" " + "is not a supported test execution style.", + option, optarg); + tr->tr_style = run_style; + tr->tr_commandline_flags |= TRF_EXECUTION_STYLE; + break; + case 't': /* Test selection option. */ + if ((selector = parse_selection_option(optarg)) == NULL) + errx(EX_USAGE, "option -%c: argument \"%s\" " + "is not a valid selection pattern.", + option, optarg); + STAILQ_INSERT_TAIL(&selections, selector, so_next); + break; + case 'v': + tr->tr_verbosity++; + break; + case ':': + errx(EX_USAGE, + "ERROR: option -%c requires an argument.", optopt); + break; + case '?': + errx(EX_USAGE, + "ERROR: unrecognized option -%c", optopt); + break; + default: + errx(EX_USAGE, "ERROR: unspecified error."); + break; + } + } + + /* + * Set unset fields of the test run descriptor to their + * defaults. + */ + if (!test_driver_finish_run_initialization(tr, argv[0])) + err(EX_OSERR, "cannot initialize test driver"); + + /* Choose tests and test cases to act upon. */ + select_tests(tr, &selections); + + assert(STAILQ_EMPTY(&selections)); + + show_run_header(tr); + + /* Perform the requested action. */ + switch (tr->tr_action) { + case TEST_RUN_LIST: + exit_code = show_listing(tr); + break; + + case TEST_RUN_EXECUTE: + default: + /* Not yet implemented. */ + exit_code = EX_UNAVAILABLE; + } + + show_run_trailer(tr); + + test_driver_free_run(tr); + + exit(exit_code); +} diff --git a/test/libtest/driver/test_driver.1 b/test/libtest/driver/test_driver.1 new file mode 100644 index 000000000000..a987002e8c67 --- /dev/null +++ b/test/libtest/driver/test_driver.1 @@ -0,0 +1,308 @@ +.\" Copyright (c) 2019 Joseph Koshy. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" This software is provided by Joseph Koshy ``as is'' and +.\" any express or implied warranties, including, but not limited to, the +.\" implied warranties of merchantability and fitness for a particular purpose +.\" are disclaimed. in no event shall Joseph Koshy be liable +.\" for any direct, indirect, incidental, special, exemplary, or consequential +.\" damages (including, but not limited to, procurement of substitute goods +.\" or services; loss of use, data, or profits; or business interruption) +.\" however caused and on any theory of liability, whether in contract, strict +.\" liability, or tort (including negligence or otherwise) arising in any way +.\" out of the use of this software, even if advised of the possibility of +.\" such damage. +.\" +.\" $Id$ +.\" +.Dd April 02, 2019 +.Dt TEST-DRIVER 1 +.Os +.Sh NAME +.Nm test-driver +.Nd scaffolding for executing +.Xr test 3 +based tests from the command-line +.Sh SYNOPSIS +.Nm test-executable +.Op Fl c Ar artefact-archive-name +.Op Fl l +.Op Fl n Ar run-name +.Op Fl p Ar search-path-directory +.Op Fl R Ar runtime-base-directory +.Op Fl s Ar execution-style +.Op Fl t Ar test-selector +.Op Fl T Ar seconds +.Op Fl v +.Sh DESCRIPTION +The +.Nm +library provides a +.Fn main +function that will execute the +.Xr test 3 +based tests in an executable according to the options specified +on the command-line. +The +.Nm +library usually used in conjunction with code generated by the +.Xr make-test-scaffolding 1 +utility. +.Pp +Test executables built using +.Nm +recognize the following command-line options: +.Bl -tag -width indent +.It Fl c Ar archive-name +If this option is specified, then the +.Nm +provided scaffolding will copy test outputs and other artefacts from +the test run to the archive named by argument +.Ar archive-name . +The format of the archive is specified by the path name suffix of the +artefact name. +The supported output formats are those supported by +.Xr libarchive 3 . +.It Fl l +If this option is specified, then the +.Nm +utility will list the selected tests and exit. +.It Fl n Ar run-name +Use the specified test run name in status messages and to name +any files and directories created during the test run. +If this option is not specified, then the base name of the test +executable is used. +.It Fl p Ar search-path-directory +Add the argument +.Ar search-path-directory +to the list of directories searched for by +.Xr test 3 +utility functions. +.It Fl R Ar runtime-base-directory +Set the runtime base directory to the directory specified by the +argument +.Ar runtime-base-directory . +Tests execute with their current directory set to a subdirectory +within this directory. +The path specified by argument +.Ar runtime-base-directory +must exist, and must name a directory. +.Pp +If this option is not specified, then the +.Ev TEST_TMPDIR +environment variable will be examined. +If set to a non-empty value, then its value will be used. +Otherwise, the value of the +.Ev TMPDIR +environment variable will be used, if non-empty. +If neither of the environment variables +.Ev TEST_TMPDIR +and +.Ev TMPDIR +contain a non-empty value, then the path +.Dq Pa /tmp +will be used. +.It Fl s Ar execution-style +Set the desired execution style to that specified by argument +.Ar execution-style . +Legal values for +.Ar execution-style +are: +.Bl -tag -width indent -compact +.It Li atf +Be compatible with +.Nx +.Xr atf 9 . +.It Li tap +Be compatible with TAP +.Pq Test Anything Protocol . +.It Li test +Be compatible with libtest (this test framework). +.El +The default is to use libtest semantics. +.It Fl t Ar test-selector +Select (or deselect) tests to execute according to the argument +.Ar test-selector . +.Pp +Test selectors are specified using the following syntax: +.Bl -tag -compact -width indent +.It Xo +.Op Li - Ns +.Li c : Ns Ar pattern +.Xc +Select test cases whose names match +.Ar pattern . +Selecting a test case will cause all of its contained +test functions to be selected. +.It Xo +.Op Li - Ns +.Li f : Ns Ar pattern +.Xc +Select test functions whose names match +.Ar pattern . +.It Xo +.Op Li - Ns +.Li t : Ns Ar pattern +.Xc +Select the test cases and test functions associated with +tags matching +.Ar pattern . +.It Xo +.Op Li - Ns +.Ar pattern +.Xc +If the +.Li c , +.Li f +or +.Li t +qualifiers were not specified, then the pattern is matched +against the names of test cases. +.El +The +.Ar pattern +fields of test selectors use shell wildcard syntax, as implemented by +.Xr fnmatch 3 . +.Pp +If no test selectors are specified then all the tests present in +the test executable will be run. +Otherwise, the test selectors specified are processed in the +order specified on the command line. +.Pp +A test selector that does not start with a +.Dq Li - +will add the entries that it matches to the currently selected list +of tests. +A test selector that starts with a +.Dq Li - +will remove the entries that it matches from the currently selected list +of tests. +.Pp +If at least one test selector was specified, and if the result of +applying the specified test selectors was an empty list +of tests, then the +.Nm +library will exit with an error message. +.It Fl T Ar seconds +Set the timeout for individual tests to +.Ar seconds . +If a test function fails to return with the specified number of seconds +then it is treated as having failed. +The default is to wait indefinitely for the test function to complete. +.It Fl v +Increase verbosity level by 1. +The default verbosity level is 0. +.El +.Ss Link-time Pre-requisites +The +.Nm +library expects the following symbols to be present in the +test executable it is linked with: +.Pp +.Bl -tag -width indent -compact +.It Xo +.Vt struct test_case_descriptor +.Va test_cases Ns [] +.Xc +An array of test cases descriptors. +Test case descriptors described by +.Xr test_case 5 . +.It Xo +.Vt int +.Va test_case_count +.Xc +The number of entries in the +.Va test_cases +array. +.El +.Ss Test Execution +At start up, the +.Fn main +function provided by +.Nm +will select tests (and test cases) to execute, based on the test +selection options specified. +.Pp +For each selected test case, test execution proceeds as follows: +.Bl -enum -compact +.It +The runtime directory for the test case is created. +.It +The test process forks, with test execution continuing in the +child. +.It +.Pq Child +The current directory of the process is changed to the runtime +directory. +.It +.Pq Child +The test case set up function is then executed. +If this function returns an error then test case execution is +aborted. +.It +.Pq Child +Each selected test function in the test case is then executed and +its status is output to stdout (or stderr) according to the test +execution style selected. +.It +.Pq Child +The test case tear down function is then executed. +.It +If test artefacts need to be preserved, then these are +copied to the specified archive. +.It +The test's runtime directory is then deleted. +.El +.Pp +After all test cases have been attempted, the +.Fn main +function exits with the exit code appropriate for the +test execution style selected. +.Sh EXAMPLES +To run all tests in the binary named +.Pa tc_example , +copying test artefacts to a +.Xr cpio 1 +archive named +.Pa /tmp/tc_example.cpio , +use: +.Bd -literal -offset indent +tc_example -c /tmp/tc_example.cpio +.Ed +.Pp +To execute tests in the test case +.Dq tc1 +alone, use: +.Bd -literal -offset indent +tc_example -t 'c:tc1' +.Ed +.Pp +To execute tests in the test case +.Dq tc1 +but not the test functions associated with tag +.Li tag1 , +use: +.Bd -literal -offset indent +tc_example -t 'c:tc1' -t '-t:tag1' +.Ed +.Sh DIAGNOSTICS +Test programs built with the +.Nm +library will exit with an exit code of 0 if all of the selected tests +passed when run, and with a non-zero exit code if an error +occurred during test execution. +.Sh SEE ALSO +.Xr make-test-scaffolding 1 , +.Xr fnmatch 3 , +.Xr libarchive 3 , +.Xr test 3 , +.Xr test_case 5 diff --git a/test/libtest/examples/minimal_example.c b/test/libtest/examples/minimal_example.c index 4ad08b4bce06..3bc76fe66ea9 100644 --- a/test/libtest/examples/minimal_example.c +++ b/test/libtest/examples/minimal_example.c @@ -41,10 +41,16 @@ #include "test.h" /* + * Function prototypes. + */ +enum test_result tf_helloworld(test_case_state); + +/* * Function names prefixed with 'tf_' name test functions. */ enum test_result -tf_helloworld(testcase_state state) +tf_helloworld(test_case_state state) { + (void) state; return (TEST_PASS); } diff --git a/test/libtest/examples/simple_example.c b/test/libtest/examples/simple_example.c index 6a4f6697eb51..6d72f65dd451 100644 --- a/test/libtest/examples/simple_example.c +++ b/test/libtest/examples/simple_example.c @@ -31,6 +31,14 @@ #include "test.h" /* + * Function prototypes. + */ +enum test_case_status tc_setup_helloworld(test_case_state *); +enum test_case_status tc_teardown_helloworld(test_case_state); +enum test_result tf_helloworld_sayhello(test_case_state); +enum test_result tf_helloworld_saygoodbye(test_case_state); + +/* * This source defines a single test case named 'helloworld' containing a * single test function named 'sayhello' contained in that test case. At * test execution time the test case would be selectable using the tags @@ -64,35 +72,37 @@ /* * A symbol name prefixed with 'tc_description_' contains a - * test case description. The TESTCASE_DESCRIPTION macro offers + * test case description. The TEST_CASE_DESCRIPTION macro offers * a convenient way to define such symbols. In the case of the * symbol below, the test case named is 'helloworld'. */ -TESTCASE_DESCRIPTION(helloworld) = "A description for a test case."; +TEST_CASE_DESCRIPTION(helloworld) = "A description for a test case."; /* * Function names prefixed with 'tc_setup_' are assumed to be test * case set up functions. */ -enum testcase_status -tc_setup_helloworld(testcase_state *state) +enum test_case_status +tc_setup_helloworld(test_case_state *state) { - return (TESTCASE_OK); + (void) state; + return (TEST_CASE_OK); } /* * Function names prefixed with 'tc_teardown_' are assumed to be test * case tear down functions. */ -enum testcase_status -tc_teardown_helloworld(testcase_state state) +enum test_case_status +tc_teardown_helloworld(test_case_state state) { - return (TESTCASE_OK); + (void) state; + return (TEST_CASE_OK); } /* * Names prefixed with 'tc_tags_' denote the tags associated with test - * cases. The TESTCASE_TAGS macro offers a convenient way to define such + * cases. The TESTC_ASE_TAGS macro offers a convenient way to define such * symbols. * * In the example below, all test functions belonging to the test case @@ -100,7 +110,7 @@ tc_teardown_helloworld(testcase_state state) * * Tags lists are terminated by a NULL entry. */ -TESTCASE_TAGS(helloworld) = { +TEST_CASE_TAGS(helloworld) = { "tag1", "tag2", NULL @@ -110,8 +120,16 @@ TESTCASE_TAGS(helloworld) = { * Function names prefixed with 'tf_' name test functions. */ enum test_result -tf_helloworld_sayhello(testcase_state state) +tf_helloworld_sayhello(test_case_state state) +{ + (void) state; + return (TEST_PASS); +} + +enum test_result +tf_helloworld_saygoodbye(test_case_state state) { + (void) state; return (TEST_PASS); } @@ -126,6 +144,9 @@ tf_helloworld_sayhello(testcase_state state) TEST_DESCRIPTION(helloworld_sayhello) = "A description for the test function 'tf_helloworld_sayhello'."; +TEST_DESCRIPTION(helloworld_saygoodbye) = + "A description for the test function 'tf_helloworld_saygoodbye'."; + /* * Names prefixed by 'tf_tags_' contain the tags associated with * test functions. @@ -143,3 +164,8 @@ test_tags tf_tags_helloworld_sayhello = { "tag4", NULL }; + +test_tags tf_tags_helloworld_saygoodbye = { + "tag5", + NULL +}; diff --git a/test/libtest/lib/Makefile b/test/libtest/lib/Makefile index 7359ddd76a6c..e36d8fa072ba 100644 --- a/test/libtest/lib/Makefile +++ b/test/libtest/lib/Makefile @@ -4,10 +4,9 @@ TOP= ../../.. LIB= test -SRCS= test.c \ - test_runner.c +SRCS= test.c -INCS= test.h +INCS= test.h test_case.h WARNS?= 6 diff --git a/test/libtest/lib/test.3 b/test/libtest/lib/test.3 index c7045b4039c4..e170c181595a 100644 --- a/test/libtest/lib/test.3 +++ b/test/libtest/lib/test.3 @@ -1,4 +1,4 @@ -.\" Copyright (c) 2018, Joseph Koshy. +.\" Copyright (c) 2018,2019 Joseph Koshy. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -24,7 +24,7 @@ .\" .\" $Id$ .\" -.Dd December 25, 2018 +.Dd January 21, 2019 .Dt TEST 3 .Os .Sh NAME @@ -34,20 +34,20 @@ .Lb libtest .Sh SYNOPSIS .In test.h -.Ft enum testcase_status -.Fn testcase_setup "testcase_state *state" -.Ft enum testcase_status -.Fn testcase_teardown "testcase_state state" +.Ft enum test_case_status +.Fn test_case_setup "test_case_state *state" +.Ft enum test_case_status +.Fn test_case_teardown "test_case_state state" .Ft enum test_result -.Fn test_function "testcase_state state" +.Fn test_function "test_case_state state" .Vt "const char" .Va test_description [] ; .Vt "const char *" .Va test_tags [] ; .Vt "const char" -.Va testcase_description [] ; +.Va test_case_description [] ; .Vt "const char *" -.Va testcase_tags [] ; +.Va test_case_tags [] ; .Sh DESCRIPTION The .Lb libtest @@ -76,7 +76,7 @@ If specified, this set up function would be invoked prior to any test function contained in the test case. The set up function can allocate and initialize test-specific state, to be passed to test functions. -If no set up function is specified for the test case, a default no-op +If no set up function is specified for the test case, a default (no-op) function will be supplied. .It An optional test case tear down function. diff --git a/test/libtest/lib/test.h b/test/libtest/lib/test.h index 86f91463993b..6928f867a6f4 100644 --- a/test/libtest/lib/test.h +++ b/test/libtest/lib/test.h @@ -44,60 +44,59 @@ enum test_result { /* * The return values from test case set up and tear down functions. * - * - TESTCASE_OK : The set up or tear down function was successful. - * - TESTCASE_ERROR : Set up or tear down actions could not be completed. + * - TEST_CASE_OK : The set up or tear down function was successful. + * - TEST_CASE_ERROR : Set up or tear down actions could not be completed. * - * If a test case set up function returns TESTCASE_ERROR then: + * If a test case set up function returns TEST_CASE_ERROR then: * - The test functions in the test case will not be run. * - The test case's tear down function will not be invoked. * - The test run as a whole will be treated as being in error. * - * If a test case tear down function returns a TESTCASE_ERROR, then + * If a test case tear down function returns a TEST_CASE_ERROR, then * the test run as a whole be treated as being in error. */ -enum testcase_status { - TESTCASE_OK = 0, - TESTCASE_ERROR = 1 +enum test_case_status { + TEST_CASE_OK = 0, + TEST_CASE_ERROR = 1 }; /* - * A testcase_state denotes resources that are shared by the test - * functions that are part of a test case. A testcase_state is allocated - * by the set up function for a test case. Conversely the test case's - * tear down function is responsible for deallocating the resources - * allocated by the set up function. + * A 'test_case_state' is a handle to resources shared by the test functions + * that make up a test case. A test_case_state is allocated by the test case + * set up function and is deallocated by the test case tear down function. * - * The test(3) framework treats a testcase_state as an opaque value. + * The test(3) framework treats a 'test_case_state' as an opaque value. */ -typedef void *testcase_state; +typedef void *test_case_state; /* * Test case and test function descriptions, and convenience macros * to define these. */ -typedef const char testcase_description[]; +typedef const char test_case_description[]; -#if !defined(TEST_DESCRIPTION) -#define TEST_DESCRIPTION(NAME) test_description tf_description_##NAME +#if !defined(TEST_CASE_DESCRIPTION) +#define TEST_CASE_DESCRIPTION(NAME) test_case_description tc_description_##NAME #endif typedef const char test_description[]; -#if !defined(TESTCASE_DESCRIPTION) -#define TESTCASE_DESCRIPTION(NAME) testcase_description tc_description_##NAME +#if !defined(TEST_DESCRIPTION) +#define TEST_DESCRIPTION(NAME) test_description tf_description_##NAME #endif /* * Test case and test function tags, and convenience macros to define * these. */ -typedef const char *testcase_tags[]; +typedef const char *test_case_tags[]; -#if !defined(TESTCASE_TAGS) -#define TESTCASE_TAGS(NAME) testcase_tags tc_tags_##NAME +#if !defined(TEST_CASE_TAGS) +#define TEST_CASE_TAGS(NAME) test_case_tags tc_tags_##NAME #endif typedef const char *test_tags[]; + #if !defined(TEST_TAGS) #define TEST_TAGS(NAME) test_tags tf_tags_##NAME #endif @@ -108,7 +107,7 @@ typedef const char *test_tags[]; * If defined for a test case, this function will be called prior to * the execution of an of the test functions within the test cae. Test * case execution will be aborted if the function returns any value other - * than TESTCASE_OK. + * than TEST_CASE_OK. * * The function can set '*state' to a memory area holding test state to be * passed to test functions. @@ -116,8 +115,8 @@ typedef const char *test_tags[]; * If the test case does not define a set up function, then a default * no-op set up function will be used. */ -typedef enum testcase_status (test_case_setup_function) - (testcase_state *state); +typedef enum test_case_status test_case_setup_function( + test_case_state *state); /* * A test function. @@ -127,7 +126,7 @@ typedef enum testcase_status (test_case_setup_function) * its test succeeded or TEST_FAIL otherwise. In the event the test could * not be executed, it can return TEST_UNRESOLVED. */ -typedef enum test_result (test_function)(testcase_state state); +typedef enum test_result test_function(test_case_state state); /* * A test case tear down function. @@ -138,7 +137,8 @@ typedef enum test_result (test_function)(testcase_state state); * responsible for deallocating the resources that the set up function * had allocated. */ -typedef enum testcase_status (test_case_teardown_function)(testcase_state state); +typedef enum test_case_status test_case_teardown_function( + test_case_state state); #ifdef __cplusplus extern "C" { diff --git a/test/libtest/driver/test_main.c b/test/libtest/lib/test_case.h index 90e46d9a5fe2..c40b15fb69a1 100644 --- a/test/libtest/driver/test_main.c +++ b/test/libtest/lib/test_case.h @@ -24,26 +24,35 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _LIBTEST_TEST_CASE_H_ +#define _LIBTEST_TEST_CASE_H_ + +#include "test.h" + /* - * This file defines a "main" that parses command-line arguments and invokes - * the selected test cases. + * These structures describe the test cases that are linked into a + * test executable. */ -#include <sys/param.h> -#include <assert.h> -#include <stdlib.h> +/* A single test function, with its associated tags and description. */ +struct test_function_descriptor { + const char *tf_name; /* Test name. */ + const char *tf_description; /* Test description. */ + const char **tf_tags; /* The tags for the test. */ + test_function *tf_func; /* The function to invoke. */ +}; -#include "_elftc.h" -#include "test.h" -#include "test_runner.h" +/* A test case, with its associated tests. */ +struct test_case_descriptor { + const char *tc_name; /* Test case name. */ + const char *tc_description; /* Test case description. */ + const char **tc_tags; /* Any associated tags. */ + const struct test_function_descriptor *tc_tests; /* Contained tests. */ + const int tc_count; /* The number of tests. */ +}; -ELFTC_VCSID("$Id$"); +/* All test cases linked into the test binary. */ +extern struct test_case_descriptor test_cases[]; +extern const int test_case_count; -int -main(int argc, char **argv) -{ - (void) test_cases; - (void) argc; - (void) argv; - exit(0); -} +#endif /* _LIBTEST_TEST_CASE_H_ */ diff --git a/test/libtest/lib/test_runner.c b/test/libtest/lib/test_runner.c deleted file mode 100644 index 1366ff4a538e..000000000000 --- a/test/libtest/lib/test_runner.c +++ /dev/null @@ -1,31 +0,0 @@ -/*- - * Copyright (c) 2018, Joseph Koshy - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer - * in this position and unchanged. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * An implementation of a test driver for test(3) tests. - */ - -/* To be implemented. */ diff --git a/test/libtest/lib/test_runner.h b/test/libtest/lib/test_runner.h deleted file mode 100644 index cbf00f29b44d..000000000000 --- a/test/libtest/lib/test_runner.h +++ /dev/null @@ -1,118 +0,0 @@ -/*- - * Copyright (c) 2018, Joseph Koshy - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer - * in this position and unchanged. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _LIBTEST_TEST_RUNNER_H_ -#define _LIBTEST_TEST_RUNNER_H_ - -#include "test.h" - -/* - * These data structures and functions are used by test driver that - * execute tests. - */ - -/* - * The completion status for a test run: - * - * - TESTRUN_PASS : All test cases were successfully invoked and all test - * purposes in the test cases passed. - * - TESTRUN_FAIL : All test cases were successfully invoked but at least - * one test purpose reported a test failure. - * - TESTRUN_ERROR : At least one test case reported an error during its - * set up or tear down phase. - */ -enum testrun_status { - TESTRUN_PASS = 0, - TESTRUN_FAIL = 1, - TESTRUN_ERROR = 2 -}; - -/* - * A single test function, with its associated tags and description. - */ -struct test_descriptor { - const char *t_name; /* Test name. */ - const char *t_description; /* Test description. */ - const char **t_tags; /* Tags associated with the test. */ - test_function *t_func; /* The function to invoke. */ -}; - -/* - * A test case. - */ -struct test_case_descriptor { - const char *tc_name; /* Test case name. */ - const char *tc_description; /* Test case description. */ - const char **tc_tags; /* Any associated tags. */ - struct test_descriptor *tc_tests; /* The tests in this test case. */ -}; - -/* - * All test cases. - */ -extern struct test_case_descriptor test_cases[]; - -enum testrun_style { - /* Libtest semantics. */ - TESTRUN_STYLE_LIBTEST, - - /* - * Be compatible with the Test Anything Protocol - * (http://testanything.org/). - */ - TESTRUN_STYLE_TAP, - - /* Be compatible with NetBSD ATF(9). */ - TESTRUN_STYLE_ATF -}; - -/* - * Parameters for the run. - */ -struct test_run { - /* - * An optional name assigned by the user for this test run. - * - * This name is reported in test logs and is not interpreted - * by the test harness. - */ - char *testrun_name; - - /* The source directory for the run. */ - char *testrun_source_directory; - - /* The directory in which the test is executing. */ - char *testrun_test_directory; -}; - -#ifdef __cplusplus -extern "C" { -#endif -#ifdef __cplusplus -} -#endif - -#endif /* _LIBTEST_TEST_RUNNER_H_ */ |