From 725be42c8b52ef1bcee1643603682f134d56f6f1 Mon Sep 17 00:00:00 2001 From: Markus Raab Date: Sat, 16 Nov 2019 11:50:08 +0100 Subject: [PATCH] tests: run ctest twice might improve #2967 --- doc/news/2018-08-18_0.8.24.md | 2 +- doc/news/_preparation_next_release.md | 1 + scripts/CMakeLists.txt | 7 +------ scripts/dev/CMakeLists.txt | 5 +++++ scripts/{ => dev}/run_all.in | 3 ++- scripts/{ => dev}/run_checkshell.in | 3 ++- scripts/dev/run_memcheck.in | 9 +++++++++ scripts/{ => dev}/run_nocheckshell.in | 3 ++- scripts/{ => dev}/run_nokdbtests.in | 3 ++- scripts/run_memcheck.in | 8 -------- tests/CMakeLists.txt | 10 +++++----- 11 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 scripts/dev/CMakeLists.txt rename scripts/{ => dev}/run_all.in (72%) rename scripts/{ => dev}/run_checkshell.in (62%) create mode 100755 scripts/dev/run_memcheck.in rename scripts/{ => dev}/run_nocheckshell.in (66%) rename scripts/{ => dev}/run_nokdbtests.in (57%) delete mode 100755 scripts/run_memcheck.in diff --git a/doc/news/2018-08-18_0.8.24.md b/doc/news/2018-08-18_0.8.24.md index c5d9b806207..984fd553db8 100644 --- a/doc/news/2018-08-18_0.8.24.md +++ b/doc/news/2018-08-18_0.8.24.md @@ -454,7 +454,7 @@ Thanks to Michael Zronek and Vanessa Kos. - The script [`check_bashisms.sh`](https://master.libelektra.org/tests/shell/check_bashisms.sh) should now work correctly again, if the system uses the GNU version `find`. _(René Schwaiger)_ - The script [`reformat-cmake`](https://master.libelektra.org/scripts/dev/reformat-cmake) now checks if `cmake-format` works before it reformats CMake files. Thank you to Klemens Böswirth for the [detailed description of the problem](https://github.com/ElektraInitiative/libelektra/pull/1903#discussion_r189332987). _(René Schwaiger)_ -- `scripts/run_icheck` now no longer leaves the base directory of the project +- `scripts/build/run_icheck` now no longer leaves the base directory of the project when checking if the ABI changed. _(Lukas Winkler)_ - The completion for [fish](http://fishshell.com) now also suggest the `info/` meta attributes of the [file plugin](https://www.libelektra.org/plugins/file). _(René Schwaiger)_ diff --git a/doc/news/_preparation_next_release.md b/doc/news/_preparation_next_release.md index 52b47c02662..d376f772641 100644 --- a/doc/news/_preparation_next_release.md +++ b/doc/news/_preparation_next_release.md @@ -309,6 +309,7 @@ you up to date with the multi-language support provided by Elektra. - We disabled the test for the conversion engine. For more information, please take a look at [issue #3086](https://issues.libelektra.org/3086). _(René Schwaiger)_ - We disabled the test `testmod_zeromqsend` from the command `kdb run_all`, since it caused timeouts in high load scenarios. _(Mihael Pranjić)_ - The (Markdown) [Shell Recorder](../../tests/shell/shell_recorder/README.md) now prints the protocol for a failed test, even if the test modified the database permanently. _(René Schwaiger)_ +- We rerun ctest twice to ignore temporary build failures. _(Markus Raab)_ ## Build diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index ca807bf77b6..57f210c3864 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -19,12 +19,6 @@ install (PROGRAMS upgrade-bootstrap DESTINATION ${TARGET_TOOL_EXEC_FOLDER}) install (PROGRAMS kdb/mountpoint-info DESTINATION ${TARGET_TOOL_EXEC_FOLDER}) install (PROGRAMS check-env-dep DESTINATION ${TARGET_TOOL_EXEC_FOLDER}) -configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_all.in" "${CMAKE_CURRENT_BINARY_DIR}/run_all" @ONLY) -configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_memcheck.in" "${CMAKE_CURRENT_BINARY_DIR}/run_memcheck" @ONLY) -configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nokdbtests.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nokdbtests" @ONLY) -configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_checkshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_checkshell" @ONLY) -configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nocheckshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nocheckshell" @ONLY) - configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/make-source-package.in" "${CMAKE_CURRENT_BINARY_DIR}/make-source-package" @ONLY) configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/configure-firefox.in" "${CMAKE_CURRENT_BINARY_DIR}/configure-firefox" @ONLY) @@ -53,6 +47,7 @@ add_custom_target (source-package configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/randoop/randoop.in" "${CMAKE_CURRENT_BINARY_DIR}/randoop/randoop" @ONLY) file (COPY "${CMAKE_CURRENT_SOURCE_DIR}/randoop/randoopClasses.txt" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/randoop/") +add_subdirectory (dev) add_subdirectory (kdb) add_subdirectory (admin) add_subdirectory (completion) diff --git a/scripts/dev/CMakeLists.txt b/scripts/dev/CMakeLists.txt new file mode 100644 index 00000000000..b01ce08886b --- /dev/null +++ b/scripts/dev/CMakeLists.txt @@ -0,0 +1,5 @@ +configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_all.in" "${CMAKE_CURRENT_BINARY_DIR}/run_all" @ONLY) +configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_memcheck.in" "${CMAKE_CURRENT_BINARY_DIR}/run_memcheck" @ONLY) +configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nokdbtests.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nokdbtests" @ONLY) +configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_checkshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_checkshell" @ONLY) +configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nocheckshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nocheckshell" @ONLY) diff --git a/scripts/run_all.in b/scripts/dev/run_all.in similarity index 72% rename from scripts/run_all.in rename to scripts/dev/run_all.in index caf96e74908..53f392d3947 100755 --- a/scripts/run_all.in +++ b/scripts/dev/run_all.in @@ -13,4 +13,5 @@ fi # # also use `run_memcheck` to check for memory issues (tests are complementary) -ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check_external --build-config $1 +ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check_external --build-config $1" +ctest $ARGS || ctest $ARGS --rerun-failed diff --git a/scripts/run_checkshell.in b/scripts/dev/run_checkshell.in similarity index 62% rename from scripts/run_checkshell.in rename to scripts/dev/run_checkshell.in index 0ec70ba9c74..fbaec33aa11 100755 --- a/scripts/run_checkshell.in +++ b/scripts/dev/run_checkshell.in @@ -10,4 +10,5 @@ if [ $# -ne 1 ]; then exit 1 fi -ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -R testscr_check --build-config $1 +ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --rerun-failed --output-on-failure -R testscr_check --build-config $1" +ctest $ARGS || ctest $ARGS --rerun-failed diff --git a/scripts/dev/run_memcheck.in b/scripts/dev/run_memcheck.in new file mode 100755 index 00000000000..a5b18069bbe --- /dev/null +++ b/scripts/dev/run_memcheck.in @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo 'No build config. This script is supposed to run by using `@CMAKE_MAKE_PROGRAM_BASENAME@ run_memcheck`' + exit 1 +fi + +ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process -T memcheck --build-config $1 --output-on-failure -LE memleak" +ctest $ARGS || ctest $ARGS --rerun-failed diff --git a/scripts/run_nocheckshell.in b/scripts/dev/run_nocheckshell.in similarity index 66% rename from scripts/run_nocheckshell.in rename to scripts/dev/run_nocheckshell.in index 24b3439f812..0801c57c481 100755 --- a/scripts/run_nocheckshell.in +++ b/scripts/dev/run_nocheckshell.in @@ -10,4 +10,5 @@ if [ $# -ne 1 ]; then exit 1 fi -ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check --build-config $1 +ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check --build-config $1" +ctest $ARGS || ctest $ARGS --rerun-failed diff --git a/scripts/run_nokdbtests.in b/scripts/dev/run_nokdbtests.in similarity index 57% rename from scripts/run_nokdbtests.in rename to scripts/dev/run_nokdbtests.in index 28b28fb6847..b5bee479803 100755 --- a/scripts/run_nokdbtests.in +++ b/scripts/dev/run_nokdbtests.in @@ -7,4 +7,5 @@ fi # run all tests not writing to disc -ctest -j@PROCESSOR_COUNT@ -LE kdbtests --force-new-ctest-process --output-on-failure --build-config $1 +ARGS="-j@PROCESSOR_COUNT@ -LE kdbtests --force-new-ctest-process --output-on-failure --build-config $1" +ctest $ARGS || ctest $ARGS --rerun-failed diff --git a/scripts/run_memcheck.in b/scripts/run_memcheck.in deleted file mode 100755 index 590b5cede50..00000000000 --- a/scripts/run_memcheck.in +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo 'No build config. This script is supposed to run by using `@CMAKE_MAKE_PROGRAM_BASENAME@ run_memcheck`' - exit 1 -fi - -ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process -T memcheck --output-on-failure --build-config $1 -LE memleak diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d1f15cbdd4d..df77dc5405b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -66,26 +66,26 @@ if (NOT CMAKE_VERSION VERSION_LESS 3.2) endif (NOT CMAKE_VERSION VERSION_LESS 3.2) add_custom_target (run_all - COMMAND "${CMAKE_BINARY_DIR}/scripts/run_all" "$" + COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_all" "$" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" ${USES_TERMINAL}) add_custom_target (run_checkshell - COMMAND "${CMAKE_BINARY_DIR}/scripts/run_checkshell" "$" + COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_checkshell" "$" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" ${USES_TERMINAL}) add_custom_target (run_nocheckshell - COMMAND "${CMAKE_BINARY_DIR}/scripts/run_nocheckshell" "$" + COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_nocheckshell" "$" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" ${USES_TERMINAL}) add_custom_target (run_nokdbtests - COMMAND "${CMAKE_BINARY_DIR}/scripts/run_nokdbtests" "$" + COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_nokdbtests" "$" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}") add_custom_target (run_memcheck - COMMAND "${CMAKE_BINARY_DIR}/scripts/run_memcheck" "$" + COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_memcheck" "$" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}") include_directories (data)