diff options
Diffstat (limited to 'tools/test-program.cpp')
-rw-r--r-- | tools/test-program.cpp | 790 |
1 files changed, 0 insertions, 790 deletions
diff --git a/tools/test-program.cpp b/tools/test-program.cpp deleted file mode 100644 index 91ad78acb497..000000000000 --- a/tools/test-program.cpp +++ /dev/null @@ -1,790 +0,0 @@ -// -// Automated Testing Framework (atf) -// -// Copyright (c) 2007 The NetBSD Foundation, Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND -// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY -// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -extern "C" { -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/time.h> - -#include <fcntl.h> -#include <signal.h> -#include <unistd.h> -} - -#include <cassert> -#include <cerrno> -#include <cstdlib> -#include <cstring> -#include <fstream> -#include <iostream> - -#include "config_file.hpp" -#include "defs.hpp" -#include "env.hpp" -#include "fs.hpp" -#include "io.hpp" -#include "parser.hpp" -#include "process.hpp" -#include "requirements.hpp" -#include "signals.hpp" -#include "test-program.hpp" -#include "text.hpp" -#include "timers.hpp" -#include "user.hpp" - -namespace impl = tools::test_program; -namespace detail = tools::test_program::detail; - -namespace { - -typedef std::map< std::string, std::string > vars_map; - -static void -check_stream(std::ostream& os) -{ - // If we receive a signal while writing to the stream, the bad bit gets set. - // Things seem to behave fine afterwards if we clear such error condition. - // However, I'm not sure if it's safe to query errno at this point. - if (os.bad()) { - if (errno == EINTR) - os.clear(); - else - throw std::runtime_error("Failed"); - } -} - -namespace atf_tp { - -static const tools::parser::token_type eof_type = 0; -static const tools::parser::token_type nl_type = 1; -static const tools::parser::token_type text_type = 2; -static const tools::parser::token_type colon_type = 3; -static const tools::parser::token_type dblquote_type = 4; - -class tokenizer : public tools::parser::tokenizer< std::istream > { -public: - tokenizer(std::istream& is, size_t curline) : - tools::parser::tokenizer< std::istream > - (is, true, eof_type, nl_type, text_type, curline) - { - add_delim(':', colon_type); - add_quote('"', dblquote_type); - } -}; - -} // namespace atf_tp - -class metadata_reader : public detail::atf_tp_reader { - impl::test_cases_map m_tcs; - - void got_tc(const std::string& ident, const vars_map& props) - { - if (m_tcs.find(ident) != m_tcs.end()) - throw(std::runtime_error("Duplicate test case " + ident + - " in test program")); - m_tcs[ident] = props; - - if (m_tcs[ident].find("has.cleanup") == m_tcs[ident].end()) - m_tcs[ident].insert(std::make_pair("has.cleanup", "false")); - - if (m_tcs[ident].find("timeout") == m_tcs[ident].end()) - m_tcs[ident].insert(std::make_pair("timeout", "300")); - } - -public: - metadata_reader(std::istream& is) : - detail::atf_tp_reader(is) - { - } - - const impl::test_cases_map& - get_tcs(void) - const - { - return m_tcs; - } -}; - -struct get_metadata_params { - const tools::fs::path& executable; - const vars_map& config; - - get_metadata_params(const tools::fs::path& p_executable, - const vars_map& p_config) : - executable(p_executable), - config(p_config) - { - } -}; - -struct test_case_params { - const tools::fs::path& executable; - const std::string& test_case_name; - const std::string& test_case_part; - const vars_map& metadata; - const vars_map& config; - const tools::fs::path& resfile; - const tools::fs::path& workdir; - - test_case_params(const tools::fs::path& p_executable, - const std::string& p_test_case_name, - const std::string& p_test_case_part, - const vars_map& p_metadata, - const vars_map& p_config, - const tools::fs::path& p_resfile, - const tools::fs::path& p_workdir) : - executable(p_executable), - test_case_name(p_test_case_name), - test_case_part(p_test_case_part), - metadata(p_metadata), - config(p_config), - resfile(p_resfile), - workdir(p_workdir) - { - } -}; - -static -std::string -generate_timestamp(void) -{ - struct timeval tv; - if (gettimeofday(&tv, NULL) == -1) - return "0.0"; - - char buf[32]; - const int len = snprintf(buf, sizeof(buf), "%ld.%ld", - static_cast< long >(tv.tv_sec), - static_cast< long >(tv.tv_usec)); - if (len >= static_cast< int >(sizeof(buf)) || len < 0) - return "0.0"; - else - return buf; -} - -static -void -append_to_vector(std::vector< std::string >& v1, - const std::vector< std::string >& v2) -{ - std::copy(v2.begin(), v2.end(), - std::back_insert_iterator< std::vector< std::string > >(v1)); -} - -static -char** -vector_to_argv(const std::vector< std::string >& v) -{ - char** argv = new char*[v.size() + 1]; - for (std::vector< std::string >::size_type i = 0; i < v.size(); i++) { - argv[i] = strdup(v[i].c_str()); - } - argv[v.size()] = NULL; - return argv; -} - -static -void -exec_or_exit(const tools::fs::path& executable, - const std::vector< std::string >& argv) -{ - // This leaks memory in case of a failure, but it is OK. Exiting will - // do the necessary cleanup. - char* const* native_argv = vector_to_argv(argv); - - ::execv(executable.c_str(), native_argv); - - const std::string message = "Failed to execute '" + executable.str() + - "': " + std::strerror(errno) + "\n"; - if (::write(STDERR_FILENO, message.c_str(), message.length()) == -1) - std::abort(); - std::exit(EXIT_FAILURE); -} - -static -std::vector< std::string > -config_to_args(const vars_map& config) -{ - std::vector< std::string > args; - - for (vars_map::const_iterator iter = config.begin(); - iter != config.end(); iter++) - args.push_back("-v" + (*iter).first + "=" + (*iter).second); - - return args; -} - -static -void -silence_stdin(void) -{ - ::close(STDIN_FILENO); - int fd = ::open("/dev/null", O_RDONLY); - if (fd == -1) - throw std::runtime_error("Could not open /dev/null"); - assert(fd == STDIN_FILENO); -} - -static -void -prepare_child(const tools::fs::path& workdir) -{ - const int ret = ::setpgid(::getpid(), 0); - assert(ret != -1); - - ::umask(S_IWGRP | S_IWOTH); - - for (int i = 1; i <= tools::signals::last_signo; i++) - tools::signals::reset(i); - - tools::env::set("HOME", workdir.str()); - tools::env::unset("LANG"); - tools::env::unset("LC_ALL"); - tools::env::unset("LC_COLLATE"); - tools::env::unset("LC_CTYPE"); - tools::env::unset("LC_MESSAGES"); - tools::env::unset("LC_MONETARY"); - tools::env::unset("LC_NUMERIC"); - tools::env::unset("LC_TIME"); - tools::env::set("TZ", "UTC"); - - tools::env::set("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value"); - - tools::fs::change_directory(workdir); - - silence_stdin(); -} - -static -void -get_metadata_child(void* raw_params) -{ - const get_metadata_params* params = - static_cast< const get_metadata_params* >(raw_params); - - std::vector< std::string > argv; - argv.push_back(params->executable.leaf_name()); - argv.push_back("-l"); - argv.push_back("-s" + params->executable.branch_path().str()); - append_to_vector(argv, config_to_args(params->config)); - - exec_or_exit(params->executable, argv); -} - -void -run_test_case_child(void* raw_params) -{ - const test_case_params* params = - static_cast< const test_case_params* >(raw_params); - - const std::pair< int, int > user = tools::get_required_user( - params->metadata, params->config); - if (user.first != -1 && user.second != -1) - tools::user::drop_privileges(user); - - // The input 'tp' parameter may be relative and become invalid once - // we change the current working directory. - const tools::fs::path absolute_executable = params->executable.to_absolute(); - - // Prepare the test program's arguments. We use dynamic memory and - // do not care to release it. We are going to die anyway very soon, - // either due to exec(2) or to exit(3). - std::vector< std::string > argv; - argv.push_back(absolute_executable.leaf_name()); - argv.push_back("-r" + params->resfile.str()); - argv.push_back("-s" + absolute_executable.branch_path().str()); - append_to_vector(argv, config_to_args(params->config)); - argv.push_back(params->test_case_name + ":" + params->test_case_part); - - prepare_child(params->workdir); - exec_or_exit(absolute_executable, argv); -} - -static void -tokenize_result(const std::string& line, std::string& out_state, - std::string& out_arg, std::string& out_reason) -{ - const std::string::size_type pos = line.find_first_of(":("); - if (pos == std::string::npos) { - out_state = line; - out_arg = ""; - out_reason = ""; - } else if (line[pos] == ':') { - out_state = line.substr(0, pos); - out_arg = ""; - out_reason = tools::text::trim(line.substr(pos + 1)); - } else if (line[pos] == '(') { - const std::string::size_type pos2 = line.find("):", pos); - if (pos2 == std::string::npos) - throw std::runtime_error("Invalid test case result '" + line + - "': unclosed optional argument"); - out_state = line.substr(0, pos); - out_arg = line.substr(pos + 1, pos2 - pos - 1); - out_reason = tools::text::trim(line.substr(pos2 + 2)); - } else - std::abort(); -} - -static impl::test_case_result -handle_result(const std::string& state, const std::string& arg, - const std::string& reason) -{ - assert(state == "passed"); - - if (!arg.empty() || !reason.empty()) - throw std::runtime_error("The test case result '" + state + "' cannot " - "be accompanied by a reason nor an expected value"); - - return impl::test_case_result(state, -1, reason); -} - -static impl::test_case_result -handle_result_with_reason(const std::string& state, const std::string& arg, - const std::string& reason) -{ - assert(state == "expected_death" || state == "expected_failure" || - state == "expected_timeout" || state == "failed" || state == "skipped"); - - if (!arg.empty() || reason.empty()) - throw std::runtime_error("The test case result '" + state + "' must " - "be accompanied by a reason but not by an expected value"); - - return impl::test_case_result(state, -1, reason); -} - -static impl::test_case_result -handle_result_with_reason_and_arg(const std::string& state, - const std::string& arg, - const std::string& reason) -{ - assert(state == "expected_exit" || state == "expected_signal"); - - if (reason.empty()) - throw std::runtime_error("The test case result '" + state + "' must " - "be accompanied by a reason"); - - int value; - if (arg.empty()) { - value = -1; - } else { - try { - value = tools::text::to_type< int >(arg); - } catch (const std::runtime_error&) { - throw std::runtime_error("The value '" + arg + "' passed to the '" + - state + "' state must be an integer"); - } - } - - return impl::test_case_result(state, value, reason); -} - -} // anonymous namespace - -detail::atf_tp_reader::atf_tp_reader(std::istream& is) : - m_is(is) -{ -} - -detail::atf_tp_reader::~atf_tp_reader(void) -{ -} - -void -detail::atf_tp_reader::got_tc( - const std::string& ident ATF_DEFS_ATTRIBUTE_UNUSED, - const std::map< std::string, std::string >& md ATF_DEFS_ATTRIBUTE_UNUSED) -{ -} - -void -detail::atf_tp_reader::got_eof(void) -{ -} - -void -detail::atf_tp_reader::validate_and_insert(const std::string& name, - const std::string& value, const size_t lineno, - std::map< std::string, std::string >& md) -{ - using tools::parser::parse_error; - - if (value.empty()) - throw parse_error(lineno, "The value for '" + name +"' cannot be " - "empty"); - - const std::string ident_regex = "^[_A-Za-z0-9]+$"; - const std::string integer_regex = "^[0-9]+$"; - - if (name == "descr") { - // Any non-empty value is valid. - } else if (name == "has.cleanup") { - try { - (void)tools::text::to_bool(value); - } catch (const std::runtime_error&) { - throw parse_error(lineno, "The has.cleanup property requires a" - " boolean value"); - } - } else if (name == "ident") { - if (!tools::text::match(value, ident_regex)) - throw parse_error(lineno, "The identifier must match " + - ident_regex + "; was '" + value + "'"); - } else if (name == "require.arch") { - } else if (name == "require.config") { - } else if (name == "require.files") { - } else if (name == "require.machine") { - } else if (name == "require.memory") { - try { - (void)tools::text::to_bytes(value); - } catch (const std::runtime_error&) { - throw parse_error(lineno, "The require.memory property requires an " - "integer value representing an amount of bytes"); - } - } else if (name == "require.progs") { - } else if (name == "require.user") { - } else if (name == "timeout") { - if (!tools::text::match(value, integer_regex)) - throw parse_error(lineno, "The timeout property requires an integer" - " value"); - } else if (name == "use.fs") { - // Deprecated; ignore it. - } else if (name.size() > 2 && name[0] == 'X' && name[1] == '-') { - // Any non-empty value is valid. - } else { - throw parse_error(lineno, "Unknown property '" + name + "'"); - } - - md.insert(std::make_pair(name, value)); -} - -void -detail::atf_tp_reader::read(void) -{ - using tools::parser::parse_error; - using namespace atf_tp; - - std::pair< size_t, tools::parser::headers_map > hml = - tools::parser::read_headers(m_is, 1); - tools::parser::validate_content_type(hml.second, - "application/X-atf-tp", 1); - - tokenizer tkz(m_is, hml.first); - tools::parser::parser< tokenizer > p(tkz); - - try { - tools::parser::token t = p.expect(text_type, "property name"); - if (t.text() != "ident") - throw parse_error(t.lineno(), "First property of a test case " - "must be 'ident'"); - - std::map< std::string, std::string > props; - do { - const std::string name = t.text(); - t = p.expect(colon_type, "`:'"); - const std::string value = tools::text::trim(p.rest_of_line()); - t = p.expect(nl_type, "new line"); - validate_and_insert(name, value, t.lineno(), props); - - t = p.expect(eof_type, nl_type, text_type, "property name, new " - "line or eof"); - if (t.type() == nl_type || t.type() == eof_type) { - const std::map< std::string, std::string >::const_iterator - iter = props.find("ident"); - if (iter == props.end()) - throw parse_error(t.lineno(), "Test case definition did " - "not define an 'ident' property"); - ATF_PARSER_CALLBACK(p, got_tc((*iter).second, props)); - props.clear(); - - if (t.type() == nl_type) { - t = p.expect(text_type, "property name"); - if (t.text() != "ident") - throw parse_error(t.lineno(), "First property of a " - "test case must be 'ident'"); - } - } - } while (t.type() != eof_type); - ATF_PARSER_CALLBACK(p, got_eof()); - } catch (const parse_error& pe) { - p.add_error(pe); - p.reset(nl_type); - } -} - -impl::test_case_result -detail::parse_test_case_result(const std::string& line) -{ - std::string state, arg, reason; - tokenize_result(line, state, arg, reason); - - if (state == "expected_death") - return handle_result_with_reason(state, arg, reason); - else if (state.compare(0, 13, "expected_exit") == 0) - return handle_result_with_reason_and_arg(state, arg, reason); - else if (state.compare(0, 16, "expected_failure") == 0) - return handle_result_with_reason(state, arg, reason); - else if (state.compare(0, 15, "expected_signal") == 0) - return handle_result_with_reason_and_arg(state, arg, reason); - else if (state.compare(0, 16, "expected_timeout") == 0) - return handle_result_with_reason(state, arg, reason); - else if (state == "failed") - return handle_result_with_reason(state, arg, reason); - else if (state == "passed") - return handle_result(state, arg, reason); - else if (state == "skipped") - return handle_result_with_reason(state, arg, reason); - else - throw std::runtime_error("Unknown test case result type in: " + line); -} - -impl::atf_tps_writer::atf_tps_writer(std::ostream& os) : - m_os(os) -{ - tools::parser::headers_map hm; - tools::parser::attrs_map ct_attrs; - ct_attrs["version"] = "3"; - hm["Content-Type"] = - tools::parser::header_entry("Content-Type", "application/X-atf-tps", - ct_attrs); - tools::parser::write_headers(hm, m_os); -} - -void -impl::atf_tps_writer::info(const std::string& what, const std::string& val) -{ - m_os << "info: " << what << ", " << val << "\n"; - m_os.flush(); -} - -void -impl::atf_tps_writer::ntps(size_t p_ntps) -{ - m_os << "tps-count: " << p_ntps << "\n"; - m_os.flush(); -} - -void -impl::atf_tps_writer::start_tp(const std::string& tp, size_t ntcs) -{ - m_tpname = tp; - m_os << "tp-start: " << generate_timestamp() << ", " << tp << ", " - << ntcs << "\n"; - m_os.flush(); -} - -void -impl::atf_tps_writer::end_tp(const std::string& reason) -{ - assert(reason.find('\n') == std::string::npos); - if (reason.empty()) - m_os << "tp-end: " << generate_timestamp() << ", " << m_tpname << "\n"; - else - m_os << "tp-end: " << generate_timestamp() << ", " << m_tpname - << ", " << reason << "\n"; - m_os.flush(); -} - -void -impl::atf_tps_writer::start_tc(const std::string& tcname) -{ - m_tcname = tcname; - m_os << "tc-start: " << generate_timestamp() << ", " << tcname << "\n"; - m_os.flush(); -} - -void -impl::atf_tps_writer::stdout_tc(const std::string& line) -{ - m_os << "tc-so:" << line << "\n"; - check_stream(m_os); - m_os.flush(); - check_stream(m_os); -} - -void -impl::atf_tps_writer::stderr_tc(const std::string& line) -{ - m_os << "tc-se:" << line << "\n"; - check_stream(m_os); - m_os.flush(); - check_stream(m_os); -} - -void -impl::atf_tps_writer::end_tc(const std::string& state, - const std::string& reason) -{ - std::string str = ", " + m_tcname + ", " + state; - if (!reason.empty()) - str += ", " + reason; - m_os << "tc-end: " << generate_timestamp() << str << "\n"; - m_os.flush(); -} - -impl::metadata -impl::get_metadata(const tools::fs::path& executable, - const vars_map& config) -{ - get_metadata_params params(executable, config); - tools::process::child child = - tools::process::fork(get_metadata_child, - tools::process::stream_capture(), - tools::process::stream_inherit(), - static_cast< void * >(¶ms)); - - tools::io::pistream outin(child.stdout_fd()); - - metadata_reader parser(outin); - parser.read(); - - const tools::process::status status = child.wait(); - if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) - throw tools::parser::format_error("Test program returned failure " - "exit status for test case list"); - - return metadata(parser.get_tcs()); -} - -impl::test_case_result -impl::read_test_case_result(const tools::fs::path& results_path) -{ - std::ifstream results_file(results_path.c_str()); - if (!results_file) - throw std::runtime_error("Failed to open " + results_path.str()); - - std::string line, extra_line; - std::getline(results_file, line); - if (!results_file.good()) - throw std::runtime_error("Results file is empty"); - - while (std::getline(results_file, extra_line).good()) - line += "<<NEWLINE UNEXPECTED>>" + extra_line; - - results_file.close(); - - return detail::parse_test_case_result(line); -} - -namespace { - -static volatile bool terminate_poll; - -static void -sigchld_handler(const int signo ATF_DEFS_ATTRIBUTE_UNUSED) -{ - terminate_poll = true; -} - -class child_muxer : public tools::io::muxer { - impl::atf_tps_writer& m_writer; - - void - line_callback(const size_t index, const std::string& line) - { - switch (index) { - case 0: m_writer.stdout_tc(line); break; - case 1: m_writer.stderr_tc(line); break; - default: std::abort(); - } - } - -public: - child_muxer(const int* fds, const size_t nfds, - impl::atf_tps_writer& writer) : - muxer(fds, nfds), - m_writer(writer) - { - } -}; - -} // anonymous namespace - -std::pair< std::string, tools::process::status > -impl::run_test_case(const tools::fs::path& executable, - const std::string& test_case_name, - const std::string& test_case_part, - const vars_map& metadata, - const vars_map& config, - const tools::fs::path& resfile, - const tools::fs::path& workdir, - atf_tps_writer& writer) -{ - // TODO: Capture termination signals and deliver them to the subprocess - // instead. Or maybe do something else; think about it. - - test_case_params params(executable, test_case_name, test_case_part, - metadata, config, resfile, workdir); - tools::process::child child = - tools::process::fork(run_test_case_child, - tools::process::stream_capture(), - tools::process::stream_capture(), - static_cast< void * >(¶ms)); - - terminate_poll = false; - - const vars_map::const_iterator iter = metadata.find("timeout"); - assert(iter != metadata.end()); - const unsigned int timeout = - tools::text::to_type< unsigned int >((*iter).second); - const pid_t child_pid = child.pid(); - - // Get the input stream of stdout and stderr. - tools::io::file_handle outfh = child.stdout_fd(); - tools::io::file_handle errfh = child.stderr_fd(); - - bool timed_out = false; - - // Process the test case's output and multiplex it into our output - // stream as we read it. - int fds[2] = {outfh.get(), errfh.get()}; - child_muxer mux(fds, 2, writer); - try { - timers::child_timer timeout_timer(timeout, child_pid, terminate_poll); - signals::signal_programmer sigchld(SIGCHLD, sigchld_handler); - mux.mux(terminate_poll); - timed_out = timeout_timer.fired(); - } catch (...) { - std::abort(); - } - - ::killpg(child_pid, SIGKILL); - mux.flush(); - tools::process::status status = child.wait(); - - std::string reason; - - if (timed_out) { - // Don't assume the child process has been signaled due to the timeout - // expiration as older versions did. The child process may have exited - // but we may have timed out due to a subchild process getting stuck. - reason = "Test case timed out after " + tools::text::to_string(timeout) + - " " + (timeout == 1 ? "second" : "seconds"); - } - - return std::make_pair(reason, status); -} |