diff options
Diffstat (limited to 'engine/scheduler_test.cpp')
| -rw-r--r-- | engine/scheduler_test.cpp | 1242 |
1 files changed, 1242 insertions, 0 deletions
diff --git a/engine/scheduler_test.cpp b/engine/scheduler_test.cpp new file mode 100644 index 000000000000..e144761d8f01 --- /dev/null +++ b/engine/scheduler_test.cpp @@ -0,0 +1,1242 @@ +// Copyright 2014 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/scheduler.hpp" + +extern "C" { +#include <sys/types.h> + +#include <signal.h> +#include <unistd.h> +} + +#include <cstdlib> +#include <fstream> +#include <iostream> +#include <string> + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/exceptions.hpp" +#include "model/context.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/passwd.hpp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/stacktrace.hpp" +#include "utils/stream.hpp" +#include "utils/test_utils.ipp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace passwd = utils::passwd; +namespace process = utils::process; +namespace scheduler = engine::scheduler; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Checks if a string starts with a prefix. +/// +/// \param str The string to be tested. +/// \param prefix The prefix to look for. +/// +/// \return True if the string is prefixed as specified. +static bool +starts_with(const std::string& str, const std::string& prefix) +{ + return (str.length() >= prefix.length() && + str.substr(0, prefix.length()) == prefix); +} + + +/// Strips a prefix from a string and converts the rest to an integer. +/// +/// \param str The string to be tested. +/// \param prefix The prefix to strip from the string. +/// +/// \return The part of the string after the prefix converted to an integer. +static int +suffix_to_int(const std::string& str, const std::string& prefix) +{ + PRE(starts_with(str, prefix)); + try { + return text::to_type< int >(str.substr(prefix.length())); + } catch (const text::value_error& error) { + std::cerr << F("Failed: %s\n") % error.what(); + std::abort(); + } +} + + +/// Mock interface definition for testing. +/// +/// This scheduler interface does not execute external binaries. It is designed +/// to simulate the scheduler of various programs with different exit statuses. +class mock_interface : public scheduler::interface { + /// Executes the subprocess simulating an exec. + /// + /// This is just a simple wrapper over _exit(2) because we cannot use + /// std::exit on exit from this mock interface. The reason is that we do + /// not want to invoke any destructors as otherwise we'd clear up the global + /// scheduler state by mistake. This wouldn't be a major problem if it + /// wasn't because doing so deletes on-disk files and we want to leave them + /// in place so that the parent process can test for them! + /// + /// \param exit_code Exit code. + void + do_exit(const int exit_code) const UTILS_NORETURN + { + std::cout.flush(); + std::cerr.flush(); + ::_exit(exit_code); + } + + /// Executes a test case that creates various files and then fails. + void + exec_create_files_and_fail(void) const UTILS_NORETURN + { + std::cerr << "This should not be clobbered\n"; + atf::utils::create_file("first file", ""); + atf::utils::create_file("second-file", ""); + fs::mkdir_p(fs::path("dir1/dir2"), 0755); + ::kill(::getpid(), SIGTERM); + std::abort(); + } + + /// Executes a test case that deletes all files in the current directory. + /// + /// This is intended to validate that the test runs in an empty directory, + /// separate from any control files that the scheduler may have created. + void + exec_delete_all(void) const UTILS_NORETURN + { + const int exit_code = ::system("rm *") == -1 + ? EXIT_FAILURE : EXIT_SUCCESS; + + // Recreate our own cookie. + atf::utils::create_file("exec_test_was_called", ""); + + do_exit(exit_code); + } + + /// Executes a test case that returns a specific exit code. + /// + /// \param exit_code Exit status to terminate the program with. + void + exec_exit(const int exit_code) const UTILS_NORETURN + { + do_exit(exit_code); + } + + /// Executes a test case that just fails. + void + exec_fail(void) const UTILS_NORETURN + { + std::cerr << "This should not be clobbered\n"; + ::kill(::getpid(), SIGTERM); + std::abort(); + } + + /// Executes a test case that prints all input parameters to the functor. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke, which must be a + /// number. + /// \param vars User-provided variables to pass to the test program. + void + exec_print_params(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars) const + UTILS_NORETURN + { + std::cout << F("Test program: %s\n") % test_program.relative_path(); + std::cout << F("Test case: %s\n") % test_case_name; + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + std::cout << F("%s=%s\n") % (*iter).first % (*iter).second; + } + + std::cerr << F("stderr: %s\n") % test_case_name; + + do_exit(EXIT_SUCCESS); + } + +public: + /// Executes a test program's list operation. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param vars User-provided variables to pass to the test program. + void + exec_list(const model::test_program& test_program, + const config::properties_map& vars) + const UTILS_NORETURN + { + const std::string name = test_program.absolute_path().leaf_name(); + + std::cerr << name; + std::cerr.flush(); + if (name == "check_i_exist") { + if (fs::exists(test_program.absolute_path())) { + std::cout << "found\n"; + do_exit(EXIT_SUCCESS); + } else { + std::cout << "not_found\n"; + do_exit(EXIT_FAILURE); + } + } else if (name == "empty") { + do_exit(EXIT_SUCCESS); + } else if (name == "misbehave") { + utils::abort_without_coredump(); + } else if (name == "timeout") { + std::cout << "sleeping\n"; + std::cout.flush(); + ::sleep(100); + utils::abort_without_coredump(); + } else if (name == "vars") { + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + std::cout << F("%s_%s\n") % (*iter).first % (*iter).second; + } + do_exit(15); + } else { + std::abort(); + } + } + + /// Computes the test cases list of a test program. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A list of test cases. + model::test_cases_map + parse_list(const optional< process::status >& status, + const fs::path& stdout_path, + const fs::path& stderr_path) const + { + const std::string name = utils::read_file(stderr_path); + if (name == "check_i_exist") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus()); + } else if (name == "empty") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(EXIT_SUCCESS, status.get().exitstatus()); + } else if (name == "misbehave") { + throw std::runtime_error("misbehaved in parse_list"); + } else if (name == "timeout") { + ATF_REQUIRE(!status); + } else if (name == "vars") { + ATF_REQUIRE(status.get().exited()); + ATF_REQUIRE_EQ(15, status.get().exitstatus()); + } else { + ATF_FAIL("Invalid stderr contents; got " + name); + } + + model::test_cases_map_builder test_cases_builder; + + std::ifstream input(stdout_path.c_str()); + ATF_REQUIRE(input); + std::string line; + while (std::getline(input, line).good()) { + test_cases_builder.add(line); + } + + return test_cases_builder.build(); + } + + /// Executes a test case of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_program The test program to execute. + /// \param test_case_name Name of the test case to invoke. + /// \param vars User-provided variables to pass to the test program. + /// \param control_directory Directory where the interface may place control + /// files. + void + exec_test(const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& control_directory) const + { + const fs::path cookie = control_directory / "exec_test_was_called"; + std::ofstream control_file(cookie.c_str()); + if (!control_file) { + std::cerr << "Failed to create " << cookie << '\n'; + std::abort(); + } + control_file << test_case_name; + control_file.close(); + + if (test_case_name == "check_i_exist") { + do_exit(fs::exists(test_program.absolute_path()) ? 0 : 1); + } else if (starts_with(test_case_name, "cleanup_timeout")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "create_files_and_fail")) { + exec_create_files_and_fail(); + } else if (test_case_name == "delete_all") { + exec_delete_all(); + } else if (starts_with(test_case_name, "exit ")) { + exec_exit(suffix_to_int(test_case_name, "exit ")); + } else if (starts_with(test_case_name, "fail")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "print_params")) { + exec_print_params(test_program, test_case_name, vars); + } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else { + std::cerr << "Unknown test case " << test_case_name << '\n'; + std::abort(); + } + } + + /// Executes a test cleanup routine of the test program. + /// + /// This method is intended to be called within a subprocess and is expected + /// to terminate execution either by exec(2)ing the test program or by + /// exiting with a failure. + /// + /// \param test_case_name Name of the test case to invoke. + void + exec_cleanup(const model::test_program& /* test_program */, + const std::string& test_case_name, + const config::properties_map& /* vars */, + const fs::path& /* control_directory */) const + { + std::cout << "exec_cleanup was called\n"; + std::cout.flush(); + + if (starts_with(test_case_name, "cleanup_timeout")) { + ::sleep(100); + std::abort(); + } else if (starts_with(test_case_name, "fail_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "fail_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else if (starts_with(test_case_name, "pass_body_fail_cleanup")) { + exec_fail(); + } else if (starts_with(test_case_name, "skip_body_pass_cleanup")) { + exec_exit(EXIT_SUCCESS); + } else { + std::cerr << "Should not have been called for a test without " + "a cleanup routine" << '\n'; + std::abort(); + } + } + + /// Computes the result of a test case based on its termination status. + /// + /// \param status The termination status of the subprocess used to execute + /// the exec_test() method or none if the test timed out. + /// \param control_directory Path to the directory where the interface may + /// have placed control files. + /// \param stdout_path Path to the file containing the stdout of the test. + /// \param stderr_path Path to the file containing the stderr of the test. + /// + /// \return A test result. + model::test_result + compute_result(const optional< process::status >& status, + const fs::path& control_directory, + const fs::path& stdout_path, + const fs::path& stderr_path) const + { + // Do not use any ATF_* macros here. Some of the tests below invoke + // this code in a subprocess, and terminating such subprocess due to a + // failed ATF_* macro yields mysterious failures that are incredibly + // hard to debug. (Case in point: the signal_handling test is racy by + // nature, and the test run by exec_test() above may not have created + // the cookie we expect below. We don't want to "silently" exit if the + // file is not there.) + + if (!status) { + return model::test_result(model::test_result_broken, + "Timed out"); + } + + if (status.get().exited()) { + // Only sanity-check the work directory-related parameters in case + // of a clean exit. In all other cases, there is no guarantee that + // these were ever created. + const fs::path cookie = control_directory / "exec_test_was_called"; + if (!atf::utils::file_exists(cookie.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's control_directory does not seem to point " + "to the right location"); + } + const std::string test_case_name = utils::read_file(cookie); + + if (!atf::utils::file_exists(stdout_path.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's stdout_path does not exist"); + } + if (!atf::utils::file_exists(stderr_path.str())) { + return model::test_result( + model::test_result_broken, + "compute_result's stderr_path does not exist"); + } + + if (test_case_name == "skip_body_pass_cleanup") { + return model::test_result( + model::test_result_skipped, + F("Exit %s") % status.get().exitstatus()); + } else { + return model::test_result( + model::test_result_passed, + F("Exit %s") % status.get().exitstatus()); + } + } else { + return model::test_result( + model::test_result_failed, + F("Signal %s") % status.get().termsig()); + } + } +}; + + +} // anonymous namespace + + +/// Runs list_tests on the scheduler and returns the results. +/// +/// \param test_name The name of the test supported by our exec_list function. +/// \param user_config Optional user settings for the test. +/// +/// \return The loaded list of test cases. +static model::test_cases_map +check_integration_list(const char* test_name, const fs::path root, + const config::tree& user_config = engine::empty_config()) +{ + const model::test_program program = model::test_program_builder( + "mock", fs::path(test_name), root, "the-suite") + .build(); + + scheduler::scheduler_handle handle = scheduler::setup(); + const model::test_cases_map test_cases = handle.list_tests(&program, + user_config); + handle.cleanup(); + + return test_cases; +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_some); +ATF_TEST_CASE_BODY(integration__list_some) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.first", "test"); + user_config.set_string("test_suites.the-suite.second", "TEST"); + user_config.set_string("test_suites.abc.unused", "unused"); + + const model::test_cases_map test_cases = check_integration_list( + "vars", fs::path("."), user_config); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("first_test").add("second_TEST").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_check_paths); +ATF_TEST_CASE_BODY(integration__list_check_paths) +{ + fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755); + atf::utils::create_file("dir1/dir2/dir3/check_i_exist", ""); + + const model::test_cases_map test_cases = check_integration_list( + "dir2/dir3/check_i_exist", fs::path("dir1")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("found").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_timeout); +ATF_TEST_CASE_BODY(integration__list_timeout) +{ + scheduler::list_timeout = datetime::delta(1, 0); + const model::test_cases_map test_cases = check_integration_list( + "timeout", fs::path(".")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add("sleeping").build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_fail); +ATF_TEST_CASE_BODY(integration__list_fail) +{ + const model::test_cases_map test_cases = check_integration_list( + "misbehave", fs::path(".")); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_broken, + "misbehaved in parse_list"), + test_case.fake_result().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_empty); +ATF_TEST_CASE_BODY(integration__list_empty) +{ + const model::test_cases_map test_cases = check_integration_list( + "empty", fs::path(".")); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_broken, + "Empty test cases list"), + test_case.fake_result().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_one); +ATF_TEST_CASE_BODY(integration__run_one) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("exit 41").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::exec_handle exec_handle = handle.spawn_test( + program, "exit 41", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 41"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_many); +ATF_TEST_CASE_BODY(integration__run_many) +{ + static const std::size_t num_test_programs = 30; + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + // We mess around with the "current time" below, so make sure the tests do + // not spuriously exceed their deadline by bumping it to a large number. + const model::metadata infinite_timeout = model::metadata_builder() + .set_timeout(datetime::delta(1000000L, 0)).build(); + + std::size_t total_tests = 0; + std::map< scheduler::exec_handle, model::test_program_ptr > + exp_test_programs; + std::map< scheduler::exec_handle, std::string > exp_test_case_names; + std::map< scheduler::exec_handle, datetime::timestamp > exp_start_times; + std::map< scheduler::exec_handle, int > exp_exit_statuses; + for (std::size_t i = 0; i < num_test_programs; ++i) { + const std::string test_case_0 = F("exit %s") % (i * 3 + 0); + const std::string test_case_1 = F("exit %s") % (i * 3 + 1); + const std::string test_case_2 = F("exit %s") % (i * 3 + 2); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path(F("program-%s") % i), + fs::current_path(), "the-suite") + .set_metadata(infinite_timeout) + .add_test_case(test_case_0) + .add_test_case(test_case_1) + .add_test_case(test_case_2) + .build_ptr(); + + const datetime::timestamp start_time = datetime::timestamp::from_values( + 2014, 12, 8, 9, 40, 0, i); + + scheduler::exec_handle exec_handle; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_0, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_0)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3)); + ++total_tests; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_1, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_1)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 1)); + ++total_tests; + + datetime::set_mock_now(start_time); + exec_handle = handle.spawn_test(program, test_case_2, user_config); + exp_test_programs.insert(std::make_pair(exec_handle, program)); + exp_test_case_names.insert(std::make_pair(exec_handle, test_case_2)); + exp_start_times.insert(std::make_pair(exec_handle, start_time)); + exp_exit_statuses.insert(std::make_pair(exec_handle, i * 3 + 2)); + ++total_tests; + } + + for (std::size_t i = 0; i < total_tests; ++i) { + const datetime::timestamp end_time = datetime::timestamp::from_values( + 2014, 12, 8, 9, 50, 10, i); + datetime::set_mock_now(end_time); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + const scheduler::exec_handle exec_handle = + result_handle->original_pid(); + + const model::test_program_ptr test_program = exp_test_programs.find( + exec_handle)->second; + const std::string& test_case_name = exp_test_case_names.find( + exec_handle)->second; + const datetime::timestamp& start_time = exp_start_times.find( + exec_handle)->second; + const int exit_status = exp_exit_statuses.find(exec_handle)->second; + + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, + F("Exit %s") % exit_status), + test_result_handle->test_result()); + + ATF_REQUIRE_EQ(test_program, test_result_handle->test_program()); + ATF_REQUIRE_EQ(test_case_name, test_result_handle->test_case_name()); + + ATF_REQUIRE_EQ(start_time, result_handle->start_time()); + ATF_REQUIRE_EQ(end_time, result_handle->end_time()); + + result_handle->cleanup(); + + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->stdout_file().str())); + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->stderr_file().str())); + ATF_REQUIRE(!atf::utils::file_exists( + result_handle->work_directory().str())); + + result_handle.reset(); + } + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__run_check_paths); +ATF_TEST_CASE_BODY(integration__run_check_paths) +{ + fs::mkdir_p(fs::path("dir1/dir2/dir3"), 0755); + atf::utils::create_file("dir1/dir2/dir3/program", ""); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("dir2/dir3/program"), fs::path("dir1"), "the-suite") + .add_test_case("check_i_exist").build_ptr(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "check_i_exist", engine::default_config()); + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__parameters_and_output); +ATF_TEST_CASE_BODY(integration__parameters_and_output) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("print_params").build_ptr(); + + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.one", "first variable"); + user_config.set_string("test_suites.the-suite.two", "second variable"); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::exec_handle exec_handle = handle.spawn_test( + program, "print_params", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(exec_handle, result_handle->original_pid()); + ATF_REQUIRE_EQ(program, test_result_handle->test_program()); + ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + const fs::path stdout_file = result_handle->stdout_file(); + ATF_REQUIRE(atf::utils::compare_file( + stdout_file.str(), + "Test program: the-program\n" + "Test case: print_params\n" + "one=first variable\n" + "two=second variable\n")); + const fs::path stderr_file = result_handle->stderr_file(); + ATF_REQUIRE(atf::utils::compare_file( + stderr_file.str(), "stderr: print_params\n")); + + result_handle->cleanup(); + ATF_REQUIRE(!fs::exists(stdout_file)); + ATF_REQUIRE(!fs::exists(stderr_file)); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__fake_result); +ATF_TEST_CASE_BODY(integration__fake_result) +{ + const model::test_result fake_result(model::test_result_skipped, + "Some fake details"); + + model::test_cases_map test_cases; + test_cases.insert(model::test_cases_map::value_type( + "__fake__", model::test_case("__fake__", "ABC", fake_result))); + + const model::test_program_ptr program(new model::test_program( + "mock", fs::path("the-program"), fs::current_path(), "the-suite", + model::metadata_builder().build(), test_cases)); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "__fake__", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(fake_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__head_skips); +ATF_TEST_CASE_BODY(integration__cleanup__head_skips) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("skip_me", + model::metadata_builder() + .add_required_config("variable-that-does-not-exist") + .set_has_cleanup(true) + .build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "skip_me", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result( + model::test_result_skipped, + "Required configuration property " + "'variable-that-does-not-exist' not defined"), + test_result_handle->test_result()); + ATF_REQUIRE(!atf::utils::grep_file("exec_cleanup was called", + result_handle->stdout_file().str())); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +/// Runs a test to verify the behavior of cleanup routines. +/// +/// \param test_case The name of the test case to invoke. +/// \param exp_result The expected test result of the execution. +static void +do_cleanup_test(const char* test_case, + const model::test_result& exp_result) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case(test_case) + .set_metadata(model::metadata_builder().set_has_cleanup(true).build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, test_case, user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + ATF_REQUIRE(atf::utils::compare_file( + result_handle->stdout_file().str(), + "exec_cleanup was called\n")); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_skips); +ATF_TEST_CASE_BODY(integration__cleanup__body_skips) +{ + do_cleanup_test( + "skip_body_pass_cleanup", + model::test_result(model::test_result_skipped, "Exit 0")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_ok); +ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_ok) +{ + do_cleanup_test( + "fail_body_pass_cleanup", + model::test_result(model::test_result_failed, "Signal 15")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_ok__cleanup_bad); +ATF_TEST_CASE_BODY(integration__cleanup__body_ok__cleanup_bad) +{ + do_cleanup_test( + "pass_body_fail_cleanup", + model::test_result(model::test_result_broken, "Test case cleanup " + "did not terminate successfully")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__body_bad__cleanup_bad); +ATF_TEST_CASE_BODY(integration__cleanup__body_bad__cleanup_bad) +{ + do_cleanup_test( + "fail_body_fail_cleanup", + model::test_result(model::test_result_failed, "Signal 15")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__cleanup__timeout); +ATF_TEST_CASE_BODY(integration__cleanup__timeout) +{ + scheduler::cleanup_timeout = datetime::delta(1, 0); + do_cleanup_test( + "cleanup_timeout", + model::test_result(model::test_result_broken, "Test case cleanup " + "timed out")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__check_requirements); +ATF_TEST_CASE_BODY(integration__check_requirements) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("exit 12") + .set_metadata(model::metadata_builder() + .add_required_config("abcde").build()) + .build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "exit 12", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result( + model::test_result_skipped, + "Required configuration property 'abcde' not defined"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__stacktrace); +ATF_TEST_CASE_BODY(integration__stacktrace) +{ + utils::prepare_coredump_test(this); + + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("unknown-dumps-core").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "unknown-dumps-core", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_failed, + F("Signal %s") % SIGABRT), + test_result_handle->test_result()); + ATF_REQUIRE(!atf::utils::grep_file("attempting to gather stack trace", + result_handle->stdout_file().str())); + ATF_REQUIRE( atf::utils::grep_file("attempting to gather stack trace", + result_handle->stderr_file().str())); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +/// Runs a test to verify the dumping of the list of existing files on failure. +/// +/// \param test_case The name of the test case to invoke. +/// \param exp_stderr Expected contents of stderr. +static void +do_check_list_files_on_failure(const char* test_case, const char* exp_stderr) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case(test_case).build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, test_case, user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + atf::utils::cat_file(result_handle->stdout_file().str(), "child stdout: "); + ATF_REQUIRE(atf::utils::compare_file(result_handle->stdout_file().str(), + "")); + atf::utils::cat_file(result_handle->stderr_file().str(), "child stderr: "); + ATF_REQUIRE(atf::utils::compare_file(result_handle->stderr_file().str(), + exp_stderr)); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__none); +ATF_TEST_CASE_BODY(integration__list_files_on_failure__none) +{ + do_check_list_files_on_failure("fail", "This should not be clobbered\n"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__list_files_on_failure__some); +ATF_TEST_CASE_BODY(integration__list_files_on_failure__some) +{ + do_check_list_files_on_failure( + "create_files_and_fail", + "This should not be clobbered\n" + "Files left in work directory after failure: " + "dir1, first file, second-file\n"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(integration__prevent_clobbering_control_files); +ATF_TEST_CASE_BODY(integration__prevent_clobbering_control_files) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("delete_all").build_ptr(); + + const config::tree user_config = engine::empty_config(); + + scheduler::scheduler_handle handle = scheduler::setup(); + + (void)handle.spawn_test(program, "delete_all", user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(debug_test); +ATF_TEST_CASE_BODY(debug_test) +{ + const model::test_program_ptr program = model::test_program_builder( + "mock", fs::path("the-program"), fs::current_path(), "the-suite") + .add_test_case("print_params").build_ptr(); + + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.one", "first variable"); + user_config.set_string("test_suites.the-suite.two", "second variable"); + + scheduler::scheduler_handle handle = scheduler::setup(); + + const fs::path stdout_file("custom-stdout.txt"); + const fs::path stderr_file("custom-stderr.txt"); + + scheduler::result_handle_ptr result_handle = handle.debug_test( + program, "print_params", user_config, stdout_file, stderr_file); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + + ATF_REQUIRE_EQ(program, test_result_handle->test_program()); + ATF_REQUIRE_EQ("print_params", test_result_handle->test_case_name()); + ATF_REQUIRE_EQ(model::test_result(model::test_result_passed, "Exit 0"), + test_result_handle->test_result()); + + // The original output went to a file. It's only an artifact of + // debug_test() that we later get a copy in our own files. + ATF_REQUIRE(stdout_file != result_handle->stdout_file()); + ATF_REQUIRE(stderr_file != result_handle->stderr_file()); + + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); + + ATF_REQUIRE(atf::utils::compare_file( + stdout_file.str(), + "Test program: the-program\n" + "Test case: print_params\n" + "one=first variable\n" + "two=second variable\n")); + ATF_REQUIRE(atf::utils::compare_file( + stderr_file.str(), "stderr: print_params\n")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(ensure_valid_interface); +ATF_TEST_CASE_BODY(ensure_valid_interface) +{ + scheduler::ensure_valid_interface("mock"); + + ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'mock2'", + scheduler::ensure_valid_interface("mock2")); + scheduler::register_interface( + "mock2", std::shared_ptr< scheduler::interface >(new mock_interface())); + scheduler::ensure_valid_interface("mock2"); + + // Standard interfaces should not be present unless registered. + ATF_REQUIRE_THROW_RE(engine::error, "Unsupported test interface 'plain'", + scheduler::ensure_valid_interface("plain")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(registered_interface_names); +ATF_TEST_CASE_BODY(registered_interface_names) +{ + std::set< std::string > exp_names; + + exp_names.insert("mock"); + ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names()); + + scheduler::register_interface( + "mock2", std::shared_ptr< scheduler::interface >(new mock_interface())); + exp_names.insert("mock2"); + ATF_REQUIRE_EQ(exp_names, scheduler::registered_interface_names()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(current_context); +ATF_TEST_CASE_BODY(current_context) +{ + const model::context context = scheduler::current_context(); + ATF_REQUIRE_EQ(fs::current_path(), context.cwd()); + ATF_REQUIRE(utils::getallenv() == context.env()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__empty); +ATF_TEST_CASE_BODY(generate_config__empty) +{ + const config::tree user_config = engine::empty_config(); + + const config::properties_map exp_props; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "missing")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__no_matches); +ATF_TEST_CASE_BODY(generate_config__no_matches) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("architecture", "foo"); + user_config.set_string("test_suites.one.var1", "value 1"); + + const config::properties_map exp_props; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "two")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(generate_config__some_matches); +ATF_TEST_CASE_BODY(generate_config__some_matches) +{ + std::vector< passwd::user > mock_users; + mock_users.push_back(passwd::user("nobody", 1234, 5678)); + passwd::set_mock_users_for_testing(mock_users); + + config::tree user_config = engine::empty_config(); + user_config.set_string("architecture", "foo"); + user_config.set_string("unprivileged_user", "nobody"); + user_config.set_string("test_suites.one.var1", "value 1"); + user_config.set_string("test_suites.two.var2", "value 2"); + + config::properties_map exp_props; + exp_props["unprivileged-user"] = "nobody"; + exp_props["var1"] = "value 1"; + + ATF_REQUIRE_EQ(exp_props, + scheduler::generate_config(user_config, "one")); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "mock", std::shared_ptr< scheduler::interface >(new mock_interface())); + + ATF_ADD_TEST_CASE(tcs, integration__list_some); + ATF_ADD_TEST_CASE(tcs, integration__list_check_paths); + ATF_ADD_TEST_CASE(tcs, integration__list_timeout); + ATF_ADD_TEST_CASE(tcs, integration__list_fail); + ATF_ADD_TEST_CASE(tcs, integration__list_empty); + + ATF_ADD_TEST_CASE(tcs, integration__run_one); + ATF_ADD_TEST_CASE(tcs, integration__run_many); + + ATF_ADD_TEST_CASE(tcs, integration__run_check_paths); + ATF_ADD_TEST_CASE(tcs, integration__parameters_and_output); + + ATF_ADD_TEST_CASE(tcs, integration__fake_result); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__head_skips); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_skips); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_ok__cleanup_bad); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_ok); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__body_bad__cleanup_bad); + ATF_ADD_TEST_CASE(tcs, integration__cleanup__timeout); + ATF_ADD_TEST_CASE(tcs, integration__check_requirements); + ATF_ADD_TEST_CASE(tcs, integration__stacktrace); + ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__none); + ATF_ADD_TEST_CASE(tcs, integration__list_files_on_failure__some); + ATF_ADD_TEST_CASE(tcs, integration__prevent_clobbering_control_files); + + ATF_ADD_TEST_CASE(tcs, debug_test); + + ATF_ADD_TEST_CASE(tcs, ensure_valid_interface); + ATF_ADD_TEST_CASE(tcs, registered_interface_names); + + ATF_ADD_TEST_CASE(tcs, current_context); + + ATF_ADD_TEST_CASE(tcs, generate_config__empty); + ATF_ADD_TEST_CASE(tcs, generate_config__no_matches); + ATF_ADD_TEST_CASE(tcs, generate_config__some_matches); +} |