From 08334c51dbb99d9ecd2bb86a2d94ed06da9e167a Mon Sep 17 00:00:00 2001 From: Brooks Davis Date: Tue, 17 Mar 2020 16:56:50 +0000 Subject: Import the kyua testing framework for infrastructure software Imported at 0.13 plus assumulated changes to git hash a685f91. Obtained from: https://github.com/jmmv/kyua Sponsored by: DARPA --- integration/Kyuafile | 16 + integration/Makefile.am.inc | 150 +++++ integration/cmd_about_test.sh | 158 +++++ integration/cmd_config_test.sh | 355 ++++++++++ integration/cmd_db_exec_test.sh | 165 +++++ integration/cmd_db_migrate_test.sh | 167 +++++ integration/cmd_debug_test.sh | 421 ++++++++++++ integration/cmd_help_test.sh | 93 +++ integration/cmd_list_test.sh | 600 +++++++++++++++++ integration/cmd_report_html_test.sh | 267 ++++++++ integration/cmd_report_junit_test.sh | 300 +++++++++ integration/cmd_report_test.sh | 381 +++++++++++ integration/cmd_test_test.sh | 1071 ++++++++++++++++++++++++++++++ integration/global_test.sh | 146 ++++ integration/helpers/.gitignore | 11 + integration/helpers/Makefile.am.inc | 90 +++ integration/helpers/bad_test_program.cpp | 50 ++ integration/helpers/bogus_test_cases.cpp | 64 ++ integration/helpers/config.cpp | 58 ++ integration/helpers/dump_env.cpp | 74 +++ integration/helpers/expect_all_pass.cpp | 92 +++ integration/helpers/expect_some_fail.cpp | 94 +++ integration/helpers/interrupts.cpp | 62 ++ integration/helpers/metadata.cpp | 95 +++ integration/helpers/race.cpp | 99 +++ integration/helpers/simple_all_pass.cpp | 55 ++ integration/helpers/simple_some_fail.cpp | 53 ++ integration/utils.sh | 177 +++++ 28 files changed, 5364 insertions(+) create mode 100644 integration/Kyuafile create mode 100644 integration/Makefile.am.inc create mode 100755 integration/cmd_about_test.sh create mode 100755 integration/cmd_config_test.sh create mode 100755 integration/cmd_db_exec_test.sh create mode 100755 integration/cmd_db_migrate_test.sh create mode 100755 integration/cmd_debug_test.sh create mode 100755 integration/cmd_help_test.sh create mode 100755 integration/cmd_list_test.sh create mode 100755 integration/cmd_report_html_test.sh create mode 100755 integration/cmd_report_junit_test.sh create mode 100755 integration/cmd_report_test.sh create mode 100755 integration/cmd_test_test.sh create mode 100755 integration/global_test.sh create mode 100644 integration/helpers/.gitignore create mode 100644 integration/helpers/Makefile.am.inc create mode 100644 integration/helpers/bad_test_program.cpp create mode 100644 integration/helpers/bogus_test_cases.cpp create mode 100644 integration/helpers/config.cpp create mode 100644 integration/helpers/dump_env.cpp create mode 100644 integration/helpers/expect_all_pass.cpp create mode 100644 integration/helpers/expect_some_fail.cpp create mode 100644 integration/helpers/interrupts.cpp create mode 100644 integration/helpers/metadata.cpp create mode 100644 integration/helpers/race.cpp create mode 100644 integration/helpers/simple_all_pass.cpp create mode 100644 integration/helpers/simple_some_fail.cpp create mode 100755 integration/utils.sh (limited to 'integration') diff --git a/integration/Kyuafile b/integration/Kyuafile new file mode 100644 index 000000000000..2ebb4ec8acca --- /dev/null +++ b/integration/Kyuafile @@ -0,0 +1,16 @@ +syntax(2) + +test_suite("kyua") + +atf_test_program{name="cmd_about_test"} +atf_test_program{name="cmd_config_test"} +atf_test_program{name="cmd_db_exec_test"} +atf_test_program{name="cmd_db_migrate_test"} +atf_test_program{name="cmd_debug_test"} +atf_test_program{name="cmd_help_test"} +atf_test_program{name="cmd_list_test"} +atf_test_program{name="cmd_report_html_test"} +atf_test_program{name="cmd_report_junit_test"} +atf_test_program{name="cmd_report_test"} +atf_test_program{name="cmd_test_test"} +atf_test_program{name="global_test"} diff --git a/integration/Makefile.am.inc b/integration/Makefile.am.inc new file mode 100644 index 000000000000..cf9ad86d5730 --- /dev/null +++ b/integration/Makefile.am.inc @@ -0,0 +1,150 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if WITH_ATF +tests_integrationdir = $(pkgtestsdir)/integration + +tests_integration_DATA = integration/Kyuafile +EXTRA_DIST += $(tests_integration_DATA) + +ATF_SH_BUILD = \ + $(MKDIR_P) integration; \ + echo "\#! $(ATF_SH)" >integration/$${name}; \ + echo "\# AUTOMATICALLY GENERATED FROM Makefile" >>integration/$${name}; \ + if [ -n "$${substs}" ]; then \ + cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \ + | sed "$${substs}" >>integration/$${name}; \ + else \ + cat $(srcdir)/integration/utils.sh $(srcdir)/integration/$${name}.sh \ + >>integration/$${name}; \ + fi; \ + chmod +x integration/$${name} + +ATF_SH_DEPS = \ + $(srcdir)/integration/utils.sh \ + Makefile + +EXTRA_DIST += integration/utils.sh + +tests_integration_SCRIPTS = integration/cmd_about_test +CLEANFILES += integration/cmd_about_test +EXTRA_DIST += integration/cmd_about_test.sh +integration/cmd_about_test: $(srcdir)/integration/cmd_about_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_about_test"; \ + substs='s,__KYUA_DOCDIR__,$(docdir),g'; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_config_test +CLEANFILES += integration/cmd_config_test +EXTRA_DIST += integration/cmd_config_test.sh +integration/cmd_config_test: $(srcdir)/integration/cmd_config_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_config_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_db_exec_test +CLEANFILES += integration/cmd_db_exec_test +EXTRA_DIST += integration/cmd_db_exec_test.sh +integration/cmd_db_exec_test: $(srcdir)/integration/cmd_db_exec_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_db_exec_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_db_migrate_test +CLEANFILES += integration/cmd_db_migrate_test +EXTRA_DIST += integration/cmd_db_migrate_test.sh +integration/cmd_db_migrate_test: $(srcdir)/integration/cmd_db_migrate_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_db_migrate_test"; \ + substs='s,__KYUA_STOREDIR__,$(storedir),g'; \ + substs="$${substs};s,__KYUA_STORETESTDATADIR__,$(tests_storedir),g"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_debug_test +CLEANFILES += integration/cmd_debug_test +EXTRA_DIST += integration/cmd_debug_test.sh +integration/cmd_debug_test: $(srcdir)/integration/cmd_debug_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_debug_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_help_test +CLEANFILES += integration/cmd_help_test +EXTRA_DIST += integration/cmd_help_test.sh +integration/cmd_help_test: $(srcdir)/integration/cmd_help_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_help_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_list_test +CLEANFILES += integration/cmd_list_test +EXTRA_DIST += integration/cmd_list_test.sh +integration/cmd_list_test: $(srcdir)/integration/cmd_list_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_list_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_test +CLEANFILES += integration/cmd_report_test +EXTRA_DIST += integration/cmd_report_test.sh +integration/cmd_report_test: $(srcdir)/integration/cmd_report_test.sh \ + $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_html_test +CLEANFILES += integration/cmd_report_html_test +EXTRA_DIST += integration/cmd_report_html_test.sh +integration/cmd_report_html_test: \ + $(srcdir)/integration/cmd_report_html_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_html_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_report_junit_test +CLEANFILES += integration/cmd_report_junit_test +EXTRA_DIST += integration/cmd_report_junit_test.sh +integration/cmd_report_junit_test: \ + $(srcdir)/integration/cmd_report_junit_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_report_junit_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/cmd_test_test +CLEANFILES += integration/cmd_test_test +EXTRA_DIST += integration/cmd_test_test.sh +integration/cmd_test_test: $(srcdir)/integration/cmd_test_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="cmd_test_test"; \ + $(ATF_SH_BUILD) + +tests_integration_SCRIPTS += integration/global_test +CLEANFILES += integration/global_test +EXTRA_DIST += integration/global_test.sh +integration/global_test: $(srcdir)/integration/global_test.sh $(ATF_SH_DEPS) + $(AM_V_GEN)name="global_test"; \ + $(ATF_SH_BUILD) +endif + +include integration/helpers/Makefile.am.inc diff --git a/integration/cmd_about_test.sh b/integration/cmd_about_test.sh new file mode 100755 index 000000000000..06d5da5ac4c2 --- /dev/null +++ b/integration/cmd_about_test.sh @@ -0,0 +1,158 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Location of installed documents. Used to validate the output of the about +# messages against the golden files. +: "${KYUA_DOCDIR:=__KYUA_DOCDIR__}" + + +# Common code to validate the output of all about information. +# +# \param file The name of the file with the output. +check_all() { + local file="${1}"; shift + + grep -E 'kyua .*[0-9]+\.[0-9]+' "${file}" || \ + atf_fail 'No version reported' + grep 'Copyright' "${file}" || atf_fail 'No license reported' + grep '^\*[^<>]*$' "${file}" || atf_fail 'No authors reported' + grep '^\*.*<.*@.*>$' "${file}" || atf_fail 'No contributors reported' + grep 'Homepage' "${file}" || atf_fail 'No homepage reported' +} + + +utils_test_case all_topics__installed +all_topics__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \ + "${KYUA_DOCDIR}/CONTRIBUTORS" "${KYUA_DOCDIR}/LICENSE" +} +all_topics__installed_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua about + check_all stdout +} + + +utils_test_case all_topics__override +all_topics__override_body() { + mkdir docs + echo "* Author (no email)" >docs/AUTHORS + echo "* Contributor " >docs/CONTRIBUTORS + echo "Copyright text" >docs/LICENSE + export KYUA_DOCDIR=docs + atf_check -s exit:0 -o save:stdout -e empty kyua about + check_all stdout +} + + +utils_test_case topic__authors__installed +topic__authors__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/AUTHORS" \ + "${KYUA_DOCDIR}/CONTRIBUTORS" +} +topic__authors__installed_body() { + grep -h '^\* ' "${KYUA_DOCDIR}/AUTHORS" "${KYUA_DOCDIR}/CONTRIBUTORS" \ + >expout + atf_check -s exit:0 -o file:expout -e empty kyua about authors +} + + +utils_test_case topic__authors__override +topic__authors__override_body() { + mkdir docs + echo "* Author (no email)" >docs/AUTHORS + echo "* Contributor " >docs/CONTRIBUTORS + export KYUA_DOCDIR=docs + cat docs/AUTHORS docs/CONTRIBUTORS >expout + atf_check -s exit:0 -o file:expout -e empty kyua about authors +} + + +utils_test_case topic__license__installed +topic__license__installed_head() { + atf_set "require.files" "${KYUA_DOCDIR}/LICENSE" +} +topic__license__installed_body() { + atf_check -s exit:0 -o file:"${KYUA_DOCDIR}/LICENSE" -e empty \ + kyua about license +} + + +utils_test_case topic__license__override +topic__license__override_body() { + mkdir docs + echo "Copyright text" >docs/LICENSE + export KYUA_DOCDIR=docs + atf_check -s exit:0 -o file:docs/LICENSE -e empty kyua about license +} + + +utils_test_case topic__version +topic__version_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua about version + + local lines="$(wc -l stdout | awk '{ print $1 }')" + [ "${lines}" -eq 1 ] || atf_fail "Version query returned more than one line" + + grep -E '^kyua (.*) [0-9]+\.[0-9]+$' stdout || \ + atf_fail "Invalid version message" +} + + +utils_test_case topic__invalid +topic__invalid_body() { + cat >experr <stderr <"${HOME}/.kyua/kyua.conf" <expout <"${HOME}/.kyua/kyua.conf" <expout <experr <kyua.conf <kyua.conf <.kyua/kyua.conf <kyua.conf <kyua.conf <experr <"${HOME}/.kyua/kyua.conf" <config <experr <experr <Kyuafile <expout <expout2 + atf_check -s exit:0 -o file:expout2 -e empty \ + kyua db-exec --no-headers "SELECT * FROM data ORDER BY a" +} + + +atf_init_test_cases() { + atf_add_test_case one_arg + atf_add_test_case many_args + atf_add_test_case no_args + atf_add_test_case invalid_statement + atf_add_test_case no_create_store + + atf_add_test_case results_file__default_home + atf_add_test_case results_file__explicit__ok + atf_add_test_case results_file__explicit__fail + + atf_add_test_case no_headers_flag +} diff --git a/integration/cmd_db_migrate_test.sh b/integration/cmd_db_migrate_test.sh new file mode 100755 index 000000000000..404a4e774019 --- /dev/null +++ b/integration/cmd_db_migrate_test.sh @@ -0,0 +1,167 @@ +# Copyright 2013 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Location of installed schema files. +: "${KYUA_STOREDIR:=__KYUA_STOREDIR__}" + + +# Location of installed test data files. +: "${KYUA_STORETESTDATADIR:=__KYUA_STORETESTDATADIR__}" + + +# Creates an empty old-style action database. +# +# \param ... Files that contain SQL commands to be run. +create_historical_db() { + mkdir -p "${HOME}/.kyua" + cat "${@}" | sqlite3 "${HOME}/.kyua/store.db" +} + + +# Creates an empty results file. +# +# \param ... Files that contain SQL commands to be run. +create_results_file() { + mkdir -p "${HOME}/.kyua/store" + local dbname="results.$(utils_test_suite_id)-20140718-173200-123456.db" + cat "${@}" | sqlite3 "${HOME}/.kyua/store/${dbname}" +} + + +utils_test_case upgrade__from_v1 +upgrade__from_v1_head() { + atf_set require.files \ + "${KYUA_STORETESTDATADIR}/schema_v1.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v1.sql" \ + "${KYUA_STOREDIR}/migrate_v1_v2.sql" \ + "${KYUA_STOREDIR}/migrate_v2_v3.sql" + atf_set require.progs "sqlite3" +} +upgrade__from_v1_body() { + create_historical_db "${KYUA_STORETESTDATADIR}/schema_v1.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v1.sql" + atf_check -s exit:0 -o empty -e empty kyua db-migrate + for f in \ + "results.test_suite_root.20130108-111331-000000.db" \ + "results.usr_tests.20130108-123832-000000.db" \ + "results.usr_tests.20130108-112635-000000.db" + do + [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \ + "was not created" + done + [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \ + "deleted" +} + + +utils_test_case upgrade__from_v2 +upgrade__from_v2_head() { + atf_set require.files \ + "${KYUA_STORETESTDATADIR}/schema_v2.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v2.sql" \ + "${KYUA_STOREDIR}/migrate_v2_v3.sql" + atf_set require.progs "sqlite3" +} +upgrade__from_v2_body() { + create_historical_db "${KYUA_STORETESTDATADIR}/schema_v2.sql" \ + "${KYUA_STORETESTDATADIR}/testdata_v2.sql" + atf_check -s exit:0 -o empty -e empty kyua db-migrate + for f in \ + "results.test_suite_root.20130108-111331-000000.db" \ + "results.usr_tests.20130108-123832-000000.db" \ + "results.usr_tests.20130108-112635-000000.db" + do + [ -f "${HOME}/.kyua/store/${f}" ] || atf_fail "Expected file ${f}" \ + "was not created" + done + [ ! -f "${HOME}/.kyua/store.db" ] || atf_fail "Historical database not" \ + "deleted" +} + + +utils_test_case already_up_to_date +already_up_to_date_head() { + atf_set require.files "${KYUA_STOREDIR}/schema_v3.sql" + atf_set require.progs "sqlite3" +} +already_up_to_date_body() { + create_results_file "${KYUA_STOREDIR}/schema_v3.sql" + atf_check -s exit:1 -o empty -e match:"already at schema version" \ + kyua db-migrate +} + + +utils_test_case need_upgrade +need_upgrade_head() { + atf_set require.files "${KYUA_STORETESTDATADIR}/schema_v1.sql" + atf_set require.progs "sqlite3" +} +need_upgrade_body() { + create_results_file "${KYUA_STORETESTDATADIR}/schema_v1.sql" + atf_check -s exit:2 -o empty \ + -e match:"database has schema version 1.*use db-migrate" kyua report +} + + +utils_test_case results_file__ok +results_file__ok_body() { + echo "This is not a valid database" >test.db + atf_check -s exit:1 -o empty -e match:"Migration failed" \ + kyua db-migrate --results-file ./test.db +} + + +utils_test_case results_file__fail +results_file__fail_body() { + atf_check -s exit:1 -o empty -e match:"No previous results.*test.db" \ + kyua db-migrate --results-file ./test.db +} + + +utils_test_case too_many_arguments +too_many_arguments_body() { + cat >stderr <Kyuafile <experr <Kyuafile <experr <Kyuafile <expout < passed +EOF +cat >experr <Kyuafile <expout < failed: This fails on purpose +EOF + cat >experr <Kyuafile <experr <experr <experr <Kyuafile <expout < passed +EOF + atf_check -s exit:0 -o file:expout -e empty kyua debug \ + --stdout=saved.out --stderr=saved.err single:with_cleanup + + cat >expout <experr <Kyuafile <expout < passed +EOF + atf_check -s exit:0 -o file:expout -e empty kyua debug \ + --stdout=saved.out --stderr=saved.err second:pass + + cat >expout <experr <root/Kyuafile <root/subdir/Kyuafile <expout < failed: This fails on purpose +EOF + cat >experr <Kyuafile <expout < passed +EOF + cat >experr <"my-config" <Kyuafile <Kyuafile <expout < passed +EOF +cat >experr <Kyuafile <myfile <experr <Kyuafile <Kyuafile <non_executable + + cat >experr <experr <expout < broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got '' +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua debug \ + crash_on_list:__test_cases_list__ + + # CHECK_STYLE_DISABLE + cat >expout < broken: Permission denied to run test program +EOF + # CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua debug \ + non_executable:__test_cases_list__ +} + + +atf_init_test_cases() { + atf_add_test_case no_args + atf_add_test_case many_args + atf_add_test_case one_arg__ok_pass + atf_add_test_case one_arg__ok_fail + atf_add_test_case one_arg__no_match + atf_add_test_case one_arg__no_test_case + atf_add_test_case one_arg__bad_filter + + atf_add_test_case body_and_cleanup + + atf_add_test_case stdout_stderr_flags + + atf_add_test_case args_are_relative + + atf_add_test_case only_load_used_test_programs + + atf_add_test_case config_behavior + + atf_add_test_case build_root_flag + atf_add_test_case kyuafile_flag__ok + atf_add_test_case missing_kyuafile + atf_add_test_case bogus_kyuafile + atf_add_test_case bogus_test_program +} diff --git a/integration/cmd_help_test.sh b/integration/cmd_help_test.sh new file mode 100755 index 000000000000..d8afbd0e6aba --- /dev/null +++ b/integration/cmd_help_test.sh @@ -0,0 +1,93 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case global +global_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua help + grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported' + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' + if grep -- '--show' stdout; then + atf_fail 'One option of the about subcommand appeared in the output' + fi + grep 'about *Shows detailed' stdout || atf_fail 'Commands not printed' +} + + +utils_test_case one_command +one_command_body() { + atf_check -s exit:0 -o save:stdout -e empty kyua help test + grep -E 'kyua .*[0-9]+\.[0-9]+' stdout || atf_fail 'No version reported' + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep '^Run tests' stdout || atf_fail 'No description printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' + grep -- '--kyuafile' stdout || atf_fail 'Command options not printed' + if grep 'about: Shows detailed' stdout; then + atf_fail 'Printed table of commands, but should not have done so' + fi +} + + +utils_test_case ignore_bad_config +ignore_bad_config_body() { + echo 'this is an invalid configuration file' >bad-config + atf_check -s exit:0 -o save:stdout -e empty kyua -c bad-config help + grep '^Usage: kyua' stdout || atf_fail 'No usage line printed' + grep -- '--loglevel' stdout || atf_fail 'Generic options not printed' +} + + +utils_test_case unknown_command +unknown_command_body() { + cat >stderr <stderr <Kyuafile <subdir/Kyuafile <expout <Kyuafile <subdir/Kyuafile <expout <Kyuafile <expout <Kyuafile <expout <experr <experr <Kyuafile <subdir/Kyuafile <expout <experr <experr <Kyuafile <experr <Kyuafile <expout <experr <root/Kyuafile <root/subdir/Kyuafile <expout <Kyuafile <expout <"my-config" <Kyuafile <Kyuafile <first + utils_cp_helper simple_all_pass build/first + + cat >subdir/Kyuafile <subdir/second + utils_cp_helper simple_some_fail build/subdir/second + + cat >expout <Kyuafile <myfile <expout <Kyuafile <myfile <expout <Kyuafile <subdir/Kyuafile <expout <Kyuafile <experr <Kyuafile <experr <experr <subdir/Kyuafile <experr <subdir/Kyuafile <experr <Kyuafile <Kyuafile <non_executable + + cat >expout <Kyuafile <subdir/Kyuafile <subdir/ok + +# CHECK_STYLE_DISABLE + cat >experr <Kyuafile <"${dbfile_name}" + rm stdout + + # Ensure the results of 'report-html' come from the database. + rm Kyuafile simple_all_pass simple_some_fail metadata +} + + +# Ensure a file has a set of strings. +# +# \param file The name of the file to check. +# \param ... List of strings to check. +check_in_file() { + local file="${1}"; shift + + while [ ${#} -gt 0 ]; do + echo "Checking for presence of '${1}' in ${file}" + if grep "${1}" "${file}" >/dev/null; then + : + else + atf_fail "Test case output not found in HTML page ${file}" + fi + shift + done +} + + +# Ensure a file does not have a set of strings. +# +# \param file The name of the file to check. +# \param ... List of strings to check. +check_not_in_file() { + local file="${1}"; shift + + while [ ${#} -gt 0 ]; do + echo "Checking for lack of '${1}' in ${file}" + if grep "${1}" "${file}" >/dev/null; then + atf_fail "Spurious test case output found in HTML page" + fi + shift + done +} + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + for f in \ + html/index.html \ + html/context.html \ + html/simple_all_pass_skip.html \ + html/simple_some_fail_fail.html + do + test -f "${f}" || atf_fail "Missing ${f}" + done + + atf_check -o match:"2 TESTS FAILING" cat html/index.html + + check_in_file html/simple_all_pass_skip.html \ + "This is the stdout of skip" "This is the stderr of skip" + check_not_in_file html/simple_all_pass_skip.html \ + "This is the stdout of pass" "This is the stderr of pass" \ + "This is the stdout of fail" "This is the stderr of fail" \ + "Test case did not write anything to" + + check_in_file html/simple_some_fail_fail.html \ + "This is the stdout of fail" "This is the stderr of fail" + check_not_in_file html/simple_some_fail_fail.html \ + "This is the stdout of pass" "This is the stderr of pass" \ + "This is the stdout of skip" "This is the stderr of skip" \ + "Test case did not write anything to" + + check_in_file html/metadata_one_property.html \ + "description = Does nothing but has one metadata property" + check_not_in_file html/metadata_one_property.html \ + "allowed_architectures = some-architecture" + + check_in_file html/metadata_many_properties.html \ + "allowed_architectures = some-architecture" + check_not_in_file html/metadata_many_properties.html \ + "description = Does nothing but has one metadata property" +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report-html +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o ignore -e empty kyua report-html \ + --results-file="$(cat dbfile_name1)" + grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report" + + rm -rf html + atf_check -s exit:0 -o ignore -e empty kyua report-html \ + --results-file="$(cat dbfile_name2)" + grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report" +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report-html --results-file=foo +} + + +utils_test_case force__yes +force__yes_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + test -f html/index.html || atf_fail "Expected file not created" + rm html/index.html + atf_check -s exit:0 -o ignore -e empty kyua report-html --force + test -f html/index.html || atf_fail "Expected file not created" +} + + +utils_test_case force__no +force__no_body() { + run_tests "mock1" unused_dbfile_name + + atf_check -s exit:0 -o ignore -e empty kyua report-html + test -f html/index.html || atf_fail "Expected file not created" + rm html/index.html + +cat >experr <experr + atf_check -s exit:2 -o empty -e file:experr kyua report-html \ + --results-filter=passed,foo-bar +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case force__yes + atf_add_test_case force__no + + atf_add_test_case output__explicit + + atf_add_test_case results_filter__ok + atf_add_test_case results_filter__invalid +} diff --git a/integration/cmd_report_junit_test.sh b/integration/cmd_report_junit_test.sh new file mode 100755 index 000000000000..af1a464f6004 --- /dev/null +++ b/integration/cmd_report_junit_test.sh @@ -0,0 +1,300 @@ +# Copyright 2014 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Executes a mock test suite to generate data in the database. +# +# \param mock_env The value to store in a MOCK variable in the environment. +# Use this to be able to differentiate executions by inspecting the +# context of the output. +# \param dbfile_name File to which to write the path to the generated database +# file. +run_tests() { + local mock_env="${1}"; shift + local dbfile_name="${1}"; shift + + cat >Kyuafile <"${dbfile_name}" + rm stdout + + # Ensure the results of 'report-junit' come from the database. + rm Kyuafile simple_all_pass +} + + +# Removes the contents of a properties tag from stdout. +strip_properties='awk " +BEGIN { skip = 0; } + +/<\/properties>/ { + print \"\"; + skip = 0; + next; +} + +// { + print \"\"; + print \"CONTENTS STRIPPED BY TEST\"; + skip = 1; + next; +} + +{ if (!skip) print; }"' + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + utils_install_times_wrapper + + run_tests "mock1 +this should not be seen +mock1 new line" unused_dbfile_name + + cat >expout < + + +CONTENTS STRIPPED BY TEST + + +This is the stdout of pass + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of pass + + + + +This is the stdout of skip + +Skipped result details +---------------------- + +The reason for skipping is this + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of skip + + + +EOF + atf_check -s exit:0 -o file:expout -e empty -x "kyua report-junit" \ + "| ${strip_properties}" +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report-junit +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o match:"MOCK.*mock1" -o not-match:"MOCK.*mock2" \ + -e empty kyua report-junit --results-file="$(cat dbfile_name1)" + atf_check -s exit:0 -o not-match:"MOCK.*mock1" -o match:"MOCK.*mock2" \ + -e empty kyua report-junit --results-file="$(cat dbfile_name2)" +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report-junit --results-file=foo +} + + +utils_test_case output__explicit +output__explicit_body() { + run_tests unused_mock unused_dbfile_name + + cat >report < + + +CONTENTS STRIPPED BY TEST + + +This is the stdout of pass + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of pass + + + + +This is the stdout of skip + +Skipped result details +---------------------- + +The reason for skipping is this + +Test case metadata +------------------ + +allowed_architectures is empty +allowed_platforms is empty +description is empty +has_cleanup = false +is_exclusive = false +required_configs is empty +required_disk_space = 0 +required_files is empty +required_memory = 0 +required_programs is empty +required_user is empty +timeout = 300 + +Timing information +------------------ + +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Original stderr +--------------- + +This is the stderr of skip + + + +EOF + + atf_check -s exit:0 -o file:report -e empty -x kyua report-junit \ + --output=/dev/stdout "| ${strip_properties} | ${utils_strip_times}" + atf_check -s exit:0 -o empty -e save:stderr kyua report-junit \ + --output=/dev/stderr + atf_check -s exit:0 -o file:report -x cat stderr \ + "| ${strip_properties} | ${utils_strip_times}" + + atf_check -s exit:0 -o empty -e empty kyua report-junit \ + --output=my-file + atf_check -s exit:0 -o file:report -x cat my-file \ + "| ${strip_properties} | ${utils_strip_times}" +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case output__explicit +} diff --git a/integration/cmd_report_test.sh b/integration/cmd_report_test.sh new file mode 100755 index 000000000000..18a5db386dfd --- /dev/null +++ b/integration/cmd_report_test.sh @@ -0,0 +1,381 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Executes a mock test suite to generate data in the database. +# +# \param mock_env The value to store in a MOCK variable in the environment. +# Use this to be able to differentiate executions by inspecting the +# context of the output. +# \param dbfile_name File to which to write the path to the generated database +# file. +run_tests() { + local mock_env="${1}"; shift + local dbfile_name="${1}"; shift + + cat >Kyuafile <"${dbfile_name}" + rm stdout + + # Ensure the results of 'report' come from the database. + rm Kyuafile simple_all_pass +} + + +utils_test_case default_behavior__ok +default_behavior__ok_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report + + run_tests "mock2" dbfile_name2 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name2) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report +} + + +utils_test_case default_behavior__no_store +default_behavior__no_store_body() { + echo 'kyua: E: No previous results file found for test suite' \ + "$(utils_test_suite_id)." >experr + atf_check -s exit:2 -o empty -e file:experr kyua report +} + + +utils_test_case results_file__explicit +results_file__explicit_body() { + run_tests "mock1" dbfile_name1 + run_tests "mock2" dbfile_name2 + + atf_check -s exit:0 -o match:"MOCK=mock1" -o not-match:"MOCK=mock2" \ + -e empty kyua report --results-file="$(cat dbfile_name1)" \ + --verbose + atf_check -s exit:0 -o not-match:"MOCK=mock1" -o match:"MOCK=mock2" \ + -e empty kyua report --results-file="$(cat dbfile_name2)" \ + --verbose +} + + +utils_test_case results_file__not_found +results_file__not_found_body() { + atf_check -s exit:2 -o empty -e match:"kyua: E: No previous results.*foo" \ + kyua report --results-file=foo +} + + +utils_test_case output__explicit +output__explicit_body() { + run_tests unused_mock dbfile_name + + cat >report < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + + atf_check -s exit:0 -o file:report -e empty -x kyua report \ + --output=/dev/stdout "| ${utils_strip_times_but_not_ids}" + atf_check -s exit:0 -o empty -e save:stderr kyua report \ + --output=/dev/stderr + atf_check -s exit:0 -o file:report -x cat stderr \ + "| ${utils_strip_times_but_not_ids}" + + atf_check -s exit:0 -o empty -e empty kyua report \ + --output=my-file + atf_check -s exit:0 -o file:report -x cat my-file \ + "| ${utils_strip_times_but_not_ids}" +} + + +utils_test_case filter__ok +filter__ok_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + simple_all_pass:skip +} + + +utils_test_case filter__ok_passed_excluded_by_default +filter__ok_passed_excluded_by_default_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + # Passed results are excluded by default so they are not displayed even if + # requested with a test case filter. This might be somewhat confusing... + cat >expout < Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + simple_all_pass:pass + cat >expout < Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 0 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter= simple_all_pass:pass +} + + +utils_test_case filter__no_match +filter__no_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 1 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + cat >experr <expout < Execution context +Current directory: ${real_cwd} +Environment variables: +EOF + # $_ is a bash variable. To keep our tests stable, we override its value + # below to match the hardcoded value in run_tests. + env \ + HOME="${real_cwd}" \ + MOCK="mock1 +has multiple lines +and terminates here" \ + _='fake-value' \ + "$(atf_get_srcdir)/helpers/dump_env" ' ' ' ' >>expout + cat >>expout < simple_all_pass:skip +Result: skipped: The reason for skipping is this +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Duration: S.UUUs + +Metadata: + allowed_architectures is empty + allowed_platforms is empty + description is empty + has_cleanup = false + is_exclusive = false + required_configs is empty + required_disk_space = 0 + required_files is empty + required_memory = 0 + required_programs is empty + required_user is empty + timeout = 300 + +Standard output: +This is the stdout of skip + +Standard error: +This is the stderr of skip +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Start time: YYYY-MM-DDTHH:MM:SS.ssssssZ +End time: YYYY-MM-DDTHH:MM:SS.ssssssZ +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty -x kyua report --verbose \ + "| ${utils_strip_times_but_not_ids}" +} + + +utils_test_case results_filter__empty +results_filter__empty_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report --results-filter= +} + + +utils_test_case results_filter__one +results_filter__one_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=passed +} + + +utils_test_case results_filter__multiple_all_match +results_filter__multiple_all_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Passed tests +simple_all_pass:pass -> passed [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=skipped,passed +} + + +utils_test_case results_filter__multiple_some_match +results_filter__multiple_some_match_body() { + utils_install_times_wrapper + + run_tests "mock1" dbfile_name1 + + cat >expout < Skipped tests +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +===> Summary +Results read from $(cat dbfile_name1) +Test cases: 2 total, 1 skipped, 0 expected failures, 0 broken, 0 failed +Total time: S.UUUs +EOF + atf_check -s exit:0 -o file:expout -e empty kyua report \ + --results-filter=skipped,xfail,broken,failed +} + + +atf_init_test_cases() { + atf_add_test_case default_behavior__ok + atf_add_test_case default_behavior__no_store + + atf_add_test_case results_file__explicit + atf_add_test_case results_file__not_found + + atf_add_test_case filter__ok + atf_add_test_case filter__ok_passed_excluded_by_default + atf_add_test_case filter__no_match + + atf_add_test_case verbose + + atf_add_test_case output__explicit + + atf_add_test_case results_filter__empty + atf_add_test_case results_filter__one + atf_add_test_case results_filter__multiple_all_match + atf_add_test_case results_filter__multiple_some_match +} diff --git a/integration/cmd_test_test.sh b/integration/cmd_test_test.sh new file mode 100755 index 000000000000..bc8c62daf223 --- /dev/null +++ b/integration/cmd_test_test.sh @@ -0,0 +1,1071 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +utils_test_case one_test_program__all_pass +one_test_program__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < passed [S.UUUs] +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + + utils_cp_helper simple_all_pass . + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case one_test_program__some_fail +one_test_program__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < failed: This fails on purpose [S.UUUs] +simple_some_fail:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + + utils_cp_helper simple_some_fail . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case many_test_programs__all_pass +many_test_programs__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +fourth:main -> skipped: Required file '/non-existent/foo' not found [S.UUUs] +second:pass -> passed [S.UUUs] +second:skip -> skipped: The reason for skipping is this [S.UUUs] +third:pass -> passed [S.UUUs] +third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +7/7 passed (0 failed) +EOF + + utils_cp_helper simple_all_pass first + utils_cp_helper simple_all_pass second + utils_cp_helper simple_all_pass third + echo "not executed" >fourth; chmod +x fourth + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case many_test_programs__some_fail +many_test_programs__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < failed: This fails on purpose [S.UUUs] +first:pass -> passed [S.UUUs] +fourth:main -> failed: Returned non-success exit status 76 [S.UUUs] +second:fail -> failed: This fails on purpose [S.UUUs] +second:pass -> passed [S.UUUs] +third:pass -> passed [S.UUUs] +third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +4/7 passed (3 failed) +EOF + + utils_cp_helper simple_some_fail first + utils_cp_helper simple_some_fail second + utils_cp_helper simple_all_pass third + echo '#! /bin/sh' >fourth + echo 'exit 76' >>fourth + chmod +x fourth + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case expect__all_pass +expect__all_pass_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < expected_failure: This is the reason for death [S.UUUs] +expect_all_pass:exit -> expected_failure: Exiting with correct code [S.UUUs] +expect_all_pass:failure -> expected_failure: Oh no: Forced failure [S.UUUs] +expect_all_pass:signal -> expected_failure: Exiting with correct signal [S.UUUs] +expect_all_pass:timeout -> expected_failure: This times out [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +5/5 passed (0 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper expect_all_pass . + atf_check -s exit:0 -o file:expout -e empty kyua test +} + + +utils_test_case expect__some_fail +expect__some_fail_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < failed: Test case was expected to terminate abruptly but it continued execution [S.UUUs] +expect_some_fail:exit -> failed: Test case expected to exit with code 12 but got code 34 [S.UUUs] +expect_some_fail:failure -> failed: Test case was expecting a failure but none were raised [S.UUUs] +expect_some_fail:pass -> passed [S.UUUs] +expect_some_fail:signal -> failed: Test case expected to receive signal 15 but got 9 [S.UUUs] +expect_some_fail:timeout -> failed: Test case was expected to hang but it continued execution [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/6 passed (5 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper expect_some_fail . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case premature_exit +premature_exit_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < broken: Premature exit; test case received signal 9 [S.UUUs] +bogus_test_cases:exit -> broken: Premature exit; test case exited with code 0 [S.UUUs] +bogus_test_cases:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/3 passed (2 failed) +EOF +# CHECK_STYLE_ENABLE + + utils_cp_helper bogus_test_cases . + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case no_args +no_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <subdir/Kyuafile <expout < passed [S.UUUs] +simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/simple_some_fail:fail -> failed: This fails on purpose [S.UUUs] +subdir/simple_some_fail:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +3/4 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case one_arg__subdir +one_arg__subdir_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <subdir/Kyuafile <expout < passed [S.UUUs] +subdir/simple_all_pass:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:0 -o file:expout -e empty kyua test subdir +} + + +utils_test_case one_arg__test_case +one_arg__test_case_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/1 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test first:skip +} + + +utils_test_case one_arg__test_program +one_arg__test_program_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < failed: This fails on purpose [S.UUUs] +second:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test second +} + + +utils_test_case one_arg__invalid +one_arg__invalid_body() { +cat >experr <experr <Kyuafile <subdir/Kyuafile <expout < passed [S.UUUs] +subdir/second:fail -> failed: This fails on purpose [S.UUUs] +subdir/second:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/3 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test subdir first:pass +} + + +utils_test_case many_args__invalid +many_args__invalid_body() { +cat >experr <experr <Kyuafile <expout <experr <Kyuafile <expout < passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +third:fail -> failed: This fails on purpose [S.UUUs] +third:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +3/4 passed (1 failed) +EOF + + cat >experr <root/Kyuafile <root/subdir/Kyuafile <expout < passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/fourth:fail -> failed: This fails on purpose [S.UUUs] + +Results file id is $(utils_results_id root) +Results saved to $(utils_results_file root) + +2/3 passed (1 failed) +EOF + atf_check -s exit:1 -o file:expout -e empty kyua test \ + -k "$(pwd)/root/Kyuafile" first subdir/fourth:fail +} + + +utils_test_case only_load_used_test_programs +only_load_used_test_programs_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout < passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + CREATE_COOKIE="$(pwd)/cookie"; export CREATE_COOKIE + atf_check -s exit:0 -o file:expout -e empty kyua test first + if [ -f "${CREATE_COOKIE}" ]; then + atf_fail "An unmatched test case has been executed, which harms" \ + "performance" + fi +} + + +utils_test_case config_behavior +config_behavior_body() { + cat >"my-config" <Kyuafile <Kyuafile <expout < failed: This fails on purpose [S.UUUs] +some-program:pass -> passed [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +1/2 passed (1 failed) +EOF + + atf_check -s exit:1 -o file:expout -e empty kyua test + +cat >expout <Kyuafile <Kyuafile <Kyuafile <Kyuafile <subdir/Kyuafile <expout < passed [S.UUUs] +first:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/second:pass -> passed [S.UUUs] +subdir/second:skip -> skipped: The reason for skipping is this [S.UUUs] +subdir/third:pass -> passed [S.UUUs] +subdir/third:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +6/6 passed (0 failed) +EOF + + mkdir build + mkdir build/subdir + utils_cp_helper simple_all_pass build/first + utils_cp_helper simple_all_pass build/subdir/second + utils_cp_helper simple_all_pass build/subdir/third + + atf_check -s exit:0 -o file:expout -e empty kyua test --build-root=build +} + + +utils_test_case kyuafile_flag__no_args +kyuafile_flag__no_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <myfile <expout < passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile + atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile +} + + +utils_test_case kyuafile_flag__some_args +kyuafile_flag__some_args_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <myfile <expout < passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test -k myfile sometest + cat >expout < passed [S.UUUs] +sometest:skip -> skipped: The reason for skipping is this [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +2/2 passed (0 failed) +EOF + atf_check -s exit:0 -o file:expout -e empty kyua test --kyuafile=myfile \ + sometest +} + + +utils_test_case interrupt +interrupt_body() { + cat >Kyuafile <stdout 2>stderr & + pid=${!} + echo "Kyua subprocess is PID ${pid}" + + while [ ! -f body ]; do + echo "Waiting for body to start" + sleep 1 + done + echo "Body started" + sleep 1 + + echo "Sending INT signal to ${pid}" + kill -INT ${pid} + echo "Waiting for process ${pid} to exit" + wait ${pid} + ret=${?} + sed -e 's,^,kyua stdout:,' stdout + sed -e 's,^,kyua stderr:,' stderr + echo "Process ${pid} exited" + [ ${ret} -ne 0 ] || atf_fail 'No error code reported' + + [ -f cleanup ] || atf_fail 'Cleanup part not executed after signal' + atf_expect_pass + + atf_check -s exit:0 -o ignore -e empty grep 'Signal caught' stderr + atf_check -s exit:0 -o ignore -e empty \ + grep 'kyua: E: Interrupted by signal' stderr +} + + +utils_test_case exclusive_tests +exclusive_tests_body() { + cat >Kyuafile <>Kyuafile + done + utils_cp_helper race . + + atf_check \ + -s exit:0 \ + -o match:"100/100 passed" \ + kyua \ + -v parallelism=20 \ + -v test_suites.integration.shared_file="$(pwd)/shared_file" \ + test +} + + +utils_test_case no_test_program_match +no_test_program_match_body() { + utils_install_stable_test_wrapper + + cat >Kyuafile <expout <experr <Kyuafile <expout <experr <experr <subdir/Kyuafile <experr <subdir/Kyuafile <experr <"${HOME}/.kyua/kyua.conf" <Kyuafile <Kyuafile <non_executable + +# CHECK_STYLE_DISABLE + cat >expout < broken: Invalid header for test case list; expecting Content-Type for application/X-atf-tp version 1, got '' [S.UUUs] +non_executable:__test_cases_list__ -> broken: Permission denied to run test program [S.UUUs] + +Results file id is $(utils_results_id) +Results saved to $(utils_results_file) + +0/2 passed (2 failed) +EOF +# CHECK_STYLE_ENABLE + atf_check -s exit:1 -o file:expout -e empty kyua test +} + + +utils_test_case missing_test_program +missing_test_program_body() { + cat >Kyuafile <subdir/Kyuafile <subdir/ok + +# CHECK_STYLE_DISABLE + cat >experr <experr <experr <experr < +#include +#include +#include + + +int +main(void) +{ + std::cerr << "This is not a valid test program!\n"; + + const char* cookie = std::getenv("CREATE_COOKIE"); + if (cookie != NULL && std::strlen(cookie) > 0) { + std::ofstream file(cookie); + if (!file) + std::abort(); + file << "Cookie file\n"; + file.close(); + } + + return EXIT_SUCCESS; +} diff --git a/integration/helpers/bogus_test_cases.cpp b/integration/helpers/bogus_test_cases.cpp new file mode 100644 index 000000000000..1a7c27031e1b --- /dev/null +++ b/integration/helpers/bogus_test_cases.cpp @@ -0,0 +1,64 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include +#include +} + +#include + +#include + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + ::kill(::getpid(), SIGKILL); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + std::exit(EXIT_SUCCESS); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, pass); +} diff --git a/integration/helpers/config.cpp b/integration/helpers/config.cpp new file mode 100644 index 000000000000..aa4fef291725 --- /dev/null +++ b/integration/helpers/config.cpp @@ -0,0 +1,58 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include + + +ATF_TEST_CASE(get_variable); +ATF_TEST_CASE_HEAD(get_variable) +{ + const char* output = ::getenv("CONFIG_VAR_FILE"); + if (output == NULL) { + set_md_var("require.config", "the-variable"); + } else { + if (has_config_var("the-variable")) { + atf::utils::create_file(output, get_config_var("the-variable") + + std::string("\n")); + } else { + atf::utils::create_file(output, "NOT DEFINED\n"); + } + } +} +ATF_TEST_CASE_BODY(get_variable) +{ + ATF_REQUIRE_EQ("value2", get_config_var("the-variable")); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, get_variable); +} diff --git a/integration/helpers/dump_env.cpp b/integration/helpers/dump_env.cpp new file mode 100644 index 000000000000..a2e8313a0062 --- /dev/null +++ b/integration/helpers/dump_env.cpp @@ -0,0 +1,74 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Dumps all environment variables. +// +// This helper program allows comparing the printed environment variables +// to what 'kyua report --verbose' may output. It does so by sorting the +// variables and allowing the caller to customize how the output looks +// like (indentation for each line and for continuation lines). + +#include +#include + +#include "utils/env.hpp" +#include "utils/text/operations.ipp" + +namespace text = utils::text; + + +int +main(const int argc, const char* const* const argv) +{ + if (argc != 3) { + std::cerr << "Usage: dump_env \n"; + return EXIT_FAILURE; + } + const char* prefix = argv[1]; + const char* continuation_prefix = argv[2]; + + const std::map< std::string, std::string > env = utils::getallenv(); + for (std::map< std::string, std::string >::const_iterator + iter = env.begin(); iter != env.end(); ++iter) { + const std::string& name = (*iter).first; + const std::vector< std::string > value = text::split( + (*iter).second, '\n'); + + if (value.empty()) { + std::cout << prefix << name << "=\n"; + } else { + std::cout << prefix << name << '=' << value[0] << '\n'; + for (std::vector< std::string >::size_type i = 1; + i < value.size(); ++i) { + std::cout << continuation_prefix << value[i] << '\n'; + } + } + } + + return EXIT_SUCCESS; +} diff --git a/integration/helpers/expect_all_pass.cpp b/integration/helpers/expect_all_pass.cpp new file mode 100644 index 000000000000..a7df16e3a783 --- /dev/null +++ b/integration/helpers/expect_all_pass.cpp @@ -0,0 +1,92 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include +#include +} + +#include + +#include + +#include "utils/test_utils.ipp" + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + expect_death("This is the reason for death"); + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + expect_exit(12, "Exiting with correct code"); + std::exit(12); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(failure); +ATF_TEST_CASE_BODY(failure) +{ + expect_fail("Oh no"); + fail("Forced failure"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(signal); +ATF_TEST_CASE_BODY(signal) +{ + expect_signal(SIGTERM, "Exiting with correct signal"); + ::kill(::getpid(), SIGTERM); +} + + +ATF_TEST_CASE(timeout); +ATF_TEST_CASE_HEAD(timeout) +{ + set_md_var("timeout", "1"); +} +ATF_TEST_CASE_BODY(timeout) +{ + expect_timeout("This times out"); + ::sleep(10); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, failure); + ATF_ADD_TEST_CASE(tcs, signal); + ATF_ADD_TEST_CASE(tcs, timeout); +} diff --git a/integration/helpers/expect_some_fail.cpp b/integration/helpers/expect_some_fail.cpp new file mode 100644 index 000000000000..da1a9f0ceb39 --- /dev/null +++ b/integration/helpers/expect_some_fail.cpp @@ -0,0 +1,94 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include +#include +} + +#include + +#include + + +ATF_TEST_CASE_WITHOUT_HEAD(die); +ATF_TEST_CASE_BODY(die) +{ + expect_death("Won't die"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(exit); +ATF_TEST_CASE_BODY(exit) +{ + expect_exit(12, "Invalid exit code"); + std::exit(34); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(failure); +ATF_TEST_CASE_BODY(failure) +{ + expect_fail("Does not fail"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_TEST_CASE_WITHOUT_HEAD(signal); +ATF_TEST_CASE_BODY(signal) +{ + expect_signal(SIGTERM, "Invalid signal"); + ::kill(::getpid(), SIGKILL); +} + + +ATF_TEST_CASE(timeout); +ATF_TEST_CASE_HEAD(timeout) +{ + set_md_var("timeout", "1"); +} +ATF_TEST_CASE_BODY(timeout) +{ + expect_timeout("Does not time out"); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, die); + ATF_ADD_TEST_CASE(tcs, exit); + ATF_ADD_TEST_CASE(tcs, failure); + ATF_ADD_TEST_CASE(tcs, pass); + ATF_ADD_TEST_CASE(tcs, signal); + ATF_ADD_TEST_CASE(tcs, timeout); +} diff --git a/integration/helpers/interrupts.cpp b/integration/helpers/interrupts.cpp new file mode 100644 index 000000000000..b6c5a948098c --- /dev/null +++ b/integration/helpers/interrupts.cpp @@ -0,0 +1,62 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include +} + +#include + +#include + + +ATF_TEST_CASE_WITH_CLEANUP(block_body); +ATF_TEST_CASE_HEAD(block_body) +{ + set_md_var("require.config", "body-cookie cleanup-cookie"); +} +ATF_TEST_CASE_BODY(block_body) +{ + const std::string cookie(get_config_var("body-cookie")); + std::ofstream output(cookie.c_str()); + output.close(); + for (;;) + ::pause(); +} +ATF_TEST_CASE_CLEANUP(block_body) +{ + const std::string cookie(get_config_var("cleanup-cookie")); + std::ofstream output(cookie.c_str()); + output.close(); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, block_body); +} diff --git a/integration/helpers/metadata.cpp b/integration/helpers/metadata.cpp new file mode 100644 index 000000000000..8005d7d9b68d --- /dev/null +++ b/integration/helpers/metadata.cpp @@ -0,0 +1,95 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +#include + +#include "utils/test_utils.ipp" + + +ATF_TEST_CASE_WITHOUT_HEAD(no_properties); +ATF_TEST_CASE_BODY(no_properties) +{ +} + + +ATF_TEST_CASE(one_property); +ATF_TEST_CASE_HEAD(one_property) +{ + set_md_var("descr", "Does nothing but has one metadata property"); +} +ATF_TEST_CASE_BODY(one_property) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE(many_properties); +ATF_TEST_CASE_HEAD(many_properties) +{ + set_md_var("descr", " A description with some padding"); + set_md_var("require.arch", "some-architecture"); + set_md_var("require.config", "var1 var2 var3"); + set_md_var("require.files", "/my/file1 /some/other/file"); + set_md_var("require.machine", "some-platform"); + set_md_var("require.progs", "bin1 bin2 /nonexistent/bin3"); + set_md_var("require.user", "root"); + set_md_var("X-no-meaning", "I am a custom variable"); +} +ATF_TEST_CASE_BODY(many_properties) +{ + utils::abort_without_coredump(); +} + + +ATF_TEST_CASE_WITH_CLEANUP(with_cleanup); +ATF_TEST_CASE_HEAD(with_cleanup) +{ + set_md_var("timeout", "250"); +} +ATF_TEST_CASE_BODY(with_cleanup) +{ + std::cout << "Body message to stdout\n"; + std::cerr << "Body message to stderr\n"; +} +ATF_TEST_CASE_CLEANUP(with_cleanup) +{ + std::cout << "Cleanup message to stdout\n"; + std::cerr << "Cleanup message to stderr\n"; +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, no_properties); + ATF_ADD_TEST_CASE(tcs, one_property); + ATF_ADD_TEST_CASE(tcs, many_properties); + ATF_ADD_TEST_CASE(tcs, with_cleanup); +} diff --git a/integration/helpers/race.cpp b/integration/helpers/race.cpp new file mode 100644 index 000000000000..39d4b04f3923 --- /dev/null +++ b/integration/helpers/race.cpp @@ -0,0 +1,99 @@ +// Copyright 2015 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file integration/helpers/race.cpp +/// Creates a file and reads it back, looking for races. +/// +/// This program should fail with high chances if it is called multiple times at +/// once with TEST_ENV_shared_file pointing to the same file. + +extern "C" { +#include + +#include +} + +#include +#include +#include + +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/env.hpp" +#include "utils/optional.ipp" +#include "utils/stream.hpp" + +namespace fs = utils::fs; + +using utils::optional; + + +/// Entry point to the helper test program. +/// +/// \return EXIT_SUCCESS if no race is detected; EXIT_FAILURE otherwise. +int +main(void) +{ + const optional< std::string > shared_file = utils::getenv( + "TEST_ENV_shared_file"); + if (!shared_file) { + std::cerr << "Environment variable TEST_ENV_shared_file not defined\n"; + std::exit(EXIT_FAILURE); + } + const fs::path shared_path(shared_file.get()); + + if (fs::exists(shared_path)) { + std::cerr << "Shared file already exists; created by a concurrent " + "test?"; + std::exit(EXIT_FAILURE); + } + + const std::string contents = F("%s") % ::getpid(); + + std::ofstream output(shared_path.c_str()); + if (!output) { + std::cerr << "Failed to create shared file; conflict with a concurrent " + "test?"; + std::exit(EXIT_FAILURE); + } + output << contents; + output.close(); + + ::usleep(10000); + + const std::string read_contents = utils::read_file(shared_path); + if (read_contents != contents) { + std::cerr << "Shared file contains unexpected contents; modified by a " + "concurrent test?"; + std::exit(EXIT_FAILURE); + } + + fs::unlink(shared_path); + std::exit(EXIT_SUCCESS); +} diff --git a/integration/helpers/simple_all_pass.cpp b/integration/helpers/simple_all_pass.cpp new file mode 100644 index 000000000000..4e168b4cca5f --- /dev/null +++ b/integration/helpers/simple_all_pass.cpp @@ -0,0 +1,55 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ + std::cout << "This is the stdout of pass\n"; + std::cerr << "This is the stderr of pass\n"; +} + + +ATF_TEST_CASE_WITHOUT_HEAD(skip); +ATF_TEST_CASE_BODY(skip) +{ + std::cout << "This is the stdout of skip\n"; + std::cerr << "This is the stderr of skip\n"; + skip("The reason for skipping is this"); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, pass); + ATF_ADD_TEST_CASE(tcs, skip); +} diff --git a/integration/helpers/simple_some_fail.cpp b/integration/helpers/simple_some_fail.cpp new file mode 100644 index 000000000000..909ffb6e2ee1 --- /dev/null +++ b/integration/helpers/simple_some_fail.cpp @@ -0,0 +1,53 @@ +// Copyright 2011 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include + + +ATF_TEST_CASE_WITHOUT_HEAD(fail); +ATF_TEST_CASE_BODY(fail) +{ + std::cout << "This is the stdout of fail\n"; + std::cerr << "This is the stderr of fail\n"; + fail("This fails on purpose"); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(pass); +ATF_TEST_CASE_BODY(pass) +{ +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, fail); + ATF_ADD_TEST_CASE(tcs, pass); +} diff --git a/integration/utils.sh b/integration/utils.sh new file mode 100755 index 000000000000..99565a1c9857 --- /dev/null +++ b/integration/utils.sh @@ -0,0 +1,177 @@ +# Copyright 2011 The Kyua Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google Inc. nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Subcommand to strip out the durations and timestamps in a report. +# +# This is to make the reports deterministic and thus easily testable. The +# time deltas are replaced by the fixed string S.UUU and the timestamps are +# replaced by the fixed strings YYYYMMDD.HHMMSS.ssssss and +# YYYY-MM-DDTHH:MM:SS.ssssssZ depending on their original format. +# +# This variable should be used as shown here: +# +# atf_check ... -x kyua report "| ${utils_strip_times}" +# +# Use the utils_install_times_wrapper function to create a 'kyua' wrapper +# script that automatically does this. +# CHECK_STYLE_DISABLE +utils_strip_times='sed -E \ + -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \ + -e "s,[0-9]{8}-[0-9]{6}-[0-9]{6},YYYYMMDD-HHMMSS-ssssss,g" \ + -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"' +# CHECK_STYLE_ENABLE + + +# Same as utils_strip_times but avoids stripping timestamp-based report IDs. +# +# This is to make the reports deterministic and thus easily testable. The +# time deltas are replaced by the fixed string S.UUU and the timestamps are +# replaced by the fixed string YYYY-MM-DDTHH:MM:SS.ssssssZ. +# CHECK_STYLE_DISABLE +utils_strip_times_but_not_ids='sed -E \ + -e "s,( |\[|\")[0-9][0-9]*.[0-9][0-9][0-9](s]|s|\"),\1S.UUU\2,g" \ + -e "s,[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z,YYYY-MM-DDTHH:MM:SS.ssssssZ,g"' +# CHECK_STYLE_ENABLE + + +# Computes the results id for a test suite run. +# +# The computed path is "generic" in the sense that it does not include a +# real timestamp: it only includes a placeholder. This function should be +# used along the utils_strip_times function so that the timestamps of +# the real results files are stripped out. +# +# \param path Optional path to use; if not given, use the cwd. +utils_results_id() { + local test_suite_id="$(utils_test_suite_id "${@}")" + echo "${test_suite_id}.YYYYMMDD-HHMMSS-ssssss" +} + + +# Computes the results file for a test suite run. +# +# The computed path is "generic" in the sense that it does not include a +# real timestamp: it only includes a placeholder. This function should be +# used along the utils_strip_times function so that the timestampts of the +# real results files are stripped out. +# +# \param path Optional path to use; if not given, use the cwd. +utils_results_file() { + echo "${HOME}/.kyua/store/results.$(utils_results_id "${@}").db" +} + + +# Copies a helper binary from the source directory to the work directory. +# +# \param name The name of the binary to copy. +# \param destination The target location for the binary; can be either +# a directory name or a file name. +utils_cp_helper() { + local name="${1}"; shift + local destination="${1}"; shift + + ln -s "$(atf_get_srcdir)"/helpers/"${name}" "${destination}" +} + + +# Creates a 'kyua' binary in the path that strips timing data off the output. +# +# Call this on test cases that wish to replace timing data in the *stdout* of +# Kyua with the deterministic strings. This is to be used by tests that +# validate the 'test' and 'report' subcommands. +utils_install_times_wrapper() { + [ ! -x kyua ] || return + cat >kyua <kyua.tmpout +result=\${?} +cat kyua.tmpout | ${utils_strip_times} +exit \${result} +EOF + chmod +x kyua + PATH="$(pwd):${PATH}" +} + + +# Creates a 'kyua' binary in the path that makes the output of 'test' stable. +# +# Call this on test cases that wish to replace timing data with deterministic +# strings and that need the result lines in the output to be sorted +# lexicographically. The latter hides the indeterminism caused by parallel +# execution so that the output can be verified. For these reasons, this is to +# be used exclusively by tests that validate the 'test' subcommand. +utils_install_stable_test_wrapper() { + [ ! -x kyua ] || return + cat >kyua <kyua.tmpout +result=\${?} +cat kyua.tmpout | ${utils_strip_times} >kyua.tmpout2 + +# Sort the test result lines but keep the rest intact. +grep '[^ ]*:[^ ]*' kyua.tmpout2 | sort >kyua.tmpout3 +grep -v '[^ ]*:[^ ]*' kyua.tmpout2 >kyua.tmpout4 +cat kyua.tmpout3 kyua.tmpout4 + +exit \${result} +EOF + chmod +x kyua + PATH="$(pwd):${PATH}" +} + + +# Defines a test case with a default head. +utils_test_case() { + local name="${1}"; shift + + atf_test_case "${name}" + eval "${name}_head() { + atf_set require.progs kyua + }" +} + + +# Computes the test suite identifier for results files files. +# +# \param path Optional path to use; if not given, use the cwd. +utils_test_suite_id() { + local path= + if [ ${#} -gt 0 ]; then + path="$(cd ${1} && pwd)"; shift + else + path="$(pwd)" + fi + echo "${path}" | sed -e 's,^/,,' -e 's,/,_,g' +} -- cgit v1.2.3