tests: run ctest twice

might improve #2967

doc/news/2018-08-18_0.8.24.md

@@ -454,7 +454,7 @@ Thanks to Michael Zronek and Vanessa Kos.
 - The script [`check_bashisms.sh`](https://master.libelektra.org/tests/shell/check_bashisms.sh) should now work correctly again, if the
   system uses the GNU version `find`. _(René Schwaiger)_
 - The script [`reformat-cmake`](https://master.libelektra.org/scripts/dev/reformat-cmake) now checks if `cmake-format` works before it reformats CMake files. Thank you to Klemens Böswirth for the [detailed description of the problem](https://github.com/ElektraInitiative/libelektra/pull/1903#discussion_r189332987). _(René Schwaiger)_
-- `scripts/run_icheck` now no longer leaves the base directory of the project
+- `scripts/build/run_icheck` now no longer leaves the base directory of the project
   when checking if the ABI changed. _(Lukas Winkler)_
 - The completion for [fish](http://fishshell.com) now also suggest the `info/` meta attributes of the
   [file plugin](https://www.libelektra.org/plugins/file). _(René Schwaiger)_

doc/news/_preparation_next_release.md

@@ -304,6 +304,7 @@ you up to date with the multi-language support provided by Elektra.
 - We disabled the test for the conversion engine. For more information, please take a look at [issue #3086](https://issues.libelektra.org/3086). _(René Schwaiger)_
 - We disabled the test `testmod_zeromqsend` from the command `kdb run_all`, since it caused timeouts in high load scenarios. _(Mihael Pranjić)_
 - The (Markdown) [Shell Recorder](../../tests/shell/shell_recorder/README.md) now prints the protocol for a failed test, even if the test modified the database permanently. _(René Schwaiger)_
+- We rerun ctest twice to ignore temporary build failures. _(Markus Raab)_
 
 ## Build
 

scripts/CMakeLists.txt

@@ -19,12 +19,6 @@ install (PROGRAMS upgrade-bootstrap DESTINATION ${TARGET_TOOL_EXEC_FOLDER})
 install (PROGRAMS kdb/mountpoint-info DESTINATION ${TARGET_TOOL_EXEC_FOLDER})
 install (PROGRAMS check-env-dep DESTINATION ${TARGET_TOOL_EXEC_FOLDER})
 
-configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_all.in" "${CMAKE_CURRENT_BINARY_DIR}/run_all" @ONLY)
-configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_memcheck.in" "${CMAKE_CURRENT_BINARY_DIR}/run_memcheck" @ONLY)
-configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nokdbtests.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nokdbtests" @ONLY)
-configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_checkshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_checkshell" @ONLY)
-configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nocheckshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nocheckshell" @ONLY)
-
 configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/make-source-package.in" "${CMAKE_CURRENT_BINARY_DIR}/make-source-package" @ONLY)
 
 configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/configure-firefox.in" "${CMAKE_CURRENT_BINARY_DIR}/configure-firefox" @ONLY)
@@ -53,6 +47,7 @@ add_custom_target (source-package
 configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/randoop/randoop.in" "${CMAKE_CURRENT_BINARY_DIR}/randoop/randoop" @ONLY)
 file (COPY "${CMAKE_CURRENT_SOURCE_DIR}/randoop/randoopClasses.txt" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/randoop/")
 
+add_subdirectory (dev)
 add_subdirectory (kdb)
 add_subdirectory (admin)
 add_subdirectory (completion)

scripts/dev/CMakeLists.txt

@@ -0,0 +1,5 @@
+configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_all.in" "${CMAKE_CURRENT_BINARY_DIR}/run_all" @ONLY)
+configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_memcheck.in" "${CMAKE_CURRENT_BINARY_DIR}/run_memcheck" @ONLY)
+configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nokdbtests.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nokdbtests" @ONLY)
+configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_checkshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_checkshell" @ONLY)
+configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/run_nocheckshell.in" "${CMAKE_CURRENT_BINARY_DIR}/run_nocheckshell" @ONLY)

scripts/run_all.in → scripts/dev/run_all.in

@@ -13,4 +13,5 @@ fi
 #
 # also use `run_memcheck` to check for memory issues (tests are complementary)
 
-ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check_external --build-config $1
+ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check_external --build-config $1"
+ctest $ARGS || ctest $ARGS --rerun-failed

scripts/run_checkshell.in → scripts/dev/run_checkshell.in

@@ -10,4 +10,5 @@ if [ $# -ne 1 ]; then
 	exit 1
 fi
 
-ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -R testscr_check --build-config $1
+ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --rerun-failed --output-on-failure -R testscr_check --build-config $1"
+ctest $ARGS || ctest $ARGS --rerun-failed

scripts/dev/run_memcheck.in

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]; then
+	echo 'No build config. This script is supposed to run by using `@CMAKE_MAKE_PROGRAM_BASENAME@ run_memcheck`'
+	exit 1
+fi
+
+ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process -T memcheck --build-config $1 --output-on-failure -LE memleak"
+ctest $ARGS || ctest $ARGS --rerun-failed

scripts/run_nocheckshell.in → scripts/dev/run_nocheckshell.in

@@ -10,4 +10,5 @@ if [ $# -ne 1 ]; then
 	exit 1
 fi
 
-ctest -j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check --build-config $1
+ARGS="-j@PROCESSOR_COUNT@ --force-new-ctest-process --output-on-failure -E testscr_check --build-config $1"
+ctest $ARGS || ctest $ARGS --rerun-failed

scripts/run_nokdbtests.in → scripts/dev/run_nokdbtests.in

@@ -7,4 +7,5 @@ fi
 
 # run all tests not writing to disc
 
-ctest -j@PROCESSOR_COUNT@ -LE kdbtests --force-new-ctest-process --output-on-failure --build-config $1
+ARGS="-j@PROCESSOR_COUNT@ -LE kdbtests --force-new-ctest-process --output-on-failure --build-config $1"
+ctest $ARGS || ctest $ARGS --rerun-failed

tests/CMakeLists.txt

@@ -66,26 +66,26 @@ if (NOT CMAKE_VERSION VERSION_LESS 3.2)
 endif (NOT CMAKE_VERSION VERSION_LESS 3.2)
 
 add_custom_target (run_all
-		   COMMAND "${CMAKE_BINARY_DIR}/scripts/run_all" "$<CONFIGURATION>"
+		   COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_all" "$<CONFIGURATION>"
 		   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
 		   ${USES_TERMINAL})
 
 add_custom_target (run_checkshell
-		   COMMAND "${CMAKE_BINARY_DIR}/scripts/run_checkshell" "$<CONFIGURATION>"
+		   COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_checkshell" "$<CONFIGURATION>"
 		   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
 		   ${USES_TERMINAL})
 
 add_custom_target (run_nocheckshell
-		   COMMAND "${CMAKE_BINARY_DIR}/scripts/run_nocheckshell" "$<CONFIGURATION>"
+		   COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_nocheckshell" "$<CONFIGURATION>"
 		   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
 		   ${USES_TERMINAL})
 
 add_custom_target (run_nokdbtests
-		   COMMAND "${CMAKE_BINARY_DIR}/scripts/run_nokdbtests" "$<CONFIGURATION>"
+		   COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_nokdbtests" "$<CONFIGURATION>"
 		   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
 
 add_custom_target (run_memcheck
-		   COMMAND "${CMAKE_BINARY_DIR}/scripts/run_memcheck" "$<CONFIGURATION>"
+		   COMMAND "${CMAKE_BINARY_DIR}/scripts/dev/run_memcheck" "$<CONFIGURATION>"
 		   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
 
 include_directories (data)