Report generator states for failed assertions

catchorg · Aug 27, 2022 · b45b106 · b45b106
1 parent 997a7d4
commit b45b106
Show file tree

Hide file tree

Showing 16 changed files with 2,718 additions and 5 deletions.
diff --git a/examples/311-Gen-CustomCapture.cpp b/examples/311-Gen-CustomCapture.cpp
@@ -29,7 +29,7 @@ TEST_CASE("Generate random doubles across different ranges",
     // This will take r1 by reference and r2 by value.
     // Note that there are no advantages for doing so in this example,
     // it is done only for expository purposes.
-    auto number = Catch::Generators::generate( "custom capture generator", CATCH_INTERNAL_LINEINFO,
+    auto number = Catch::Generators::generate( "custom capture generator", "custom definition", CATCH_INTERNAL_LINEINFO,
         [&r1, r2]{
             using namespace Catch::Generators;
             return makeGenerators(take(50, random(std::get<0>(r1), std::get<1>(r2))));

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -111,6 +111,7 @@ set(INTERNAL_HEADERS
     ${SOURCES_DIR}/matchers/catch_matchers_templated.hpp
     ${SOURCES_DIR}/matchers/catch_matchers_vector.hpp
     ${SOURCES_DIR}/catch_message.hpp
+    ${SOURCES_DIR}/internal/catch_generator_info.hpp
     ${SOURCES_DIR}/internal/catch_message_info.hpp
     ${SOURCES_DIR}/internal/catch_meta.hpp
     ${SOURCES_DIR}/internal/catch_move_and_forward.hpp
@@ -234,6 +235,7 @@ set(IMPL_SOURCES
     ${SOURCES_DIR}/internal/catch_errno_guard.cpp
     ${SOURCES_DIR}/internal/catch_lazy_expr.cpp
     ${SOURCES_DIR}/internal/catch_leak_detector.cpp
+    ${SOURCES_DIR}/internal/catch_generator_info.cpp
     ${SOURCES_DIR}/internal/catch_message_info.cpp
     ${SOURCES_DIR}/internal/catch_polyfills.cpp
     ${SOURCES_DIR}/internal/catch_startup_exception_registry.cpp

diff --git a/src/catch2/catch_all.hpp b/src/catch2/catch_all.hpp
@@ -67,6 +67,7 @@
 #include <catch2/internal/catch_exception_translator_registry.hpp>
 #include <catch2/internal/catch_fatal_condition_handler.hpp>
 #include <catch2/internal/catch_floating_point_helpers.hpp>
+#include <catch2/internal/catch_generator_info.hpp>
 #include <catch2/internal/catch_istream.hpp>
 #include <catch2/internal/catch_lazy_expr.hpp>
 #include <catch2/internal/catch_leak_detector.hpp>

diff --git a/src/catch2/generators/catch_generators.hpp b/src/catch2/generators/catch_generators.hpp
@@ -8,8 +8,10 @@
 #ifndef CATCH_GENERATORS_HPP_INCLUDED
 #define CATCH_GENERATORS_HPP_INCLUDED
 
+#include "catch2/interfaces/catch_interfaces_capture.hpp"
 #include <catch2/catch_tostring.hpp>
 #include <catch2/interfaces/catch_interfaces_generatortracker.hpp>
+#include <catch2/internal/catch_generator_info.hpp>
 #include <catch2/internal/catch_source_line_info.hpp>
 #include <catch2/internal/catch_stringref.hpp>
 #include <catch2/internal/catch_move_and_forward.hpp>
@@ -210,7 +212,7 @@ namespace Detail {
     // Note: The type after -> is weird, because VS2015 cannot parse
     //       the expression used in the typedef inside, when it is in
     //       return type. Yeah.
-    auto generate( StringRef generatorName, SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
+    auto generate( StringRef generatorName, StringRef definition, SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
         using UnderlyingType = typename decltype(generatorExpression())::type;
 
         IGeneratorTracker& tracker = acquireGeneratorTracker( generatorName, lineInfo );
@@ -219,6 +221,7 @@ namespace Detail {
         }
 
         auto const& generator = static_cast<IGenerator<UnderlyingType> const&>( *tracker.getGenerator() );
+        getResultCapture().trackGeneratorState(GeneratorInfo(definition, lineInfo, generator.currentElementAsString()));
         return generator.get();
     }
 
@@ -227,14 +230,17 @@ namespace Detail {
 
 #define GENERATE( ... ) \
     Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
+                                 INTERNAL_CATCH_STRINGIZE( __VA_ARGS__ ), \
                                  CATCH_INTERNAL_LINEINFO, \
                                  [ ]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
 #define GENERATE_COPY( ... ) \
     Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
+                                 INTERNAL_CATCH_STRINGIZE( __VA_ARGS__ ), \
                                  CATCH_INTERNAL_LINEINFO, \
                                  [=]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
 #define GENERATE_REF( ... ) \
     Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
+                                 INTERNAL_CATCH_STRINGIZE( __VA_ARGS__ ), \
                                  CATCH_INTERNAL_LINEINFO, \
                                  [&]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
 

diff --git a/src/catch2/interfaces/catch_interfaces_capture.hpp b/src/catch2/interfaces/catch_interfaces_capture.hpp
@@ -20,6 +20,7 @@ namespace Catch {
     struct AssertionInfo;
     struct SectionInfo;
     struct SectionEndInfo;
+    struct GeneratorInfo;
     struct MessageInfo;
     struct MessageBuilder;
     struct Counts;
@@ -43,6 +44,7 @@ namespace Catch {
         virtual void sectionEndedEarly( SectionEndInfo const& endInfo ) = 0;
 
         virtual auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
+        virtual void trackGeneratorState( GeneratorInfo info ) = 0;
 
         virtual void benchmarkPreparing( StringRef name ) = 0;
         virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0;

diff --git a/src/catch2/interfaces/catch_interfaces_reporter.cpp b/src/catch2/interfaces/catch_interfaces_reporter.cpp
@@ -5,6 +5,7 @@
 //        https://www.boost.org/LICENSE_1_0.txt)
 
 // SPDX-License-Identifier: BSL-1.0
+#include "catch2/interfaces/catch_interfaces_capture.hpp"
 #include <catch2/interfaces/catch_interfaces_reporter.hpp>
 #include <catch2/interfaces/catch_interfaces_config.hpp>
 #include <catch2/internal/catch_console_colour.hpp>
@@ -49,9 +50,11 @@ namespace Catch {
 
     AssertionStats::AssertionStats( AssertionResult const& _assertionResult,
                                     std::vector<MessageInfo> const& _infoMessages,
+                                    std::vector<GeneratorInfo> const& _generatorInfos,
                                     Totals const& _totals )
     :   assertionResult( _assertionResult ),
         infoMessages( _infoMessages ),
+        generatorInfos( _generatorInfos ),
         totals( _totals )
     {
         assertionResult.m_resultData.lazyExpression.m_transientExpression = _assertionResult.m_resultData.lazyExpression.m_transientExpression;

diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp
@@ -8,10 +8,12 @@
 #ifndef CATCH_INTERFACES_REPORTER_HPP_INCLUDED
 #define CATCH_INTERFACES_REPORTER_HPP_INCLUDED
 
+#include "catch2/interfaces/catch_interfaces_capture.hpp"
 #include <catch2/catch_section_info.hpp>
 #include <catch2/catch_totals.hpp>
 #include <catch2/catch_assertion_result.hpp>
 #include <catch2/internal/catch_message_info.hpp>
+#include <catch2/internal/catch_generator_info.hpp>
 #include <catch2/internal/catch_stringref.hpp>
 #include <catch2/internal/catch_unique_ptr.hpp>
 #include <catch2/internal/catch_move_and_forward.hpp>
@@ -65,6 +67,7 @@ namespace Catch {
     struct AssertionStats {
         AssertionStats( AssertionResult const& _assertionResult,
                         std::vector<MessageInfo> const& _infoMessages,
+                        std::vector<GeneratorInfo> const& _generatorInfos,
                         Totals const& _totals );
 
         AssertionStats( AssertionStats const& )              = default;
@@ -74,6 +77,7 @@ namespace Catch {
 
         AssertionResult assertionResult;
         std::vector<MessageInfo> infoMessages;
+        std::vector<GeneratorInfo> generatorInfos;
         Totals totals;
     };
 

diff --git a/src/catch2/internal/catch_console_colour.hpp b/src/catch2/internal/catch_console_colour.hpp
@@ -50,6 +50,7 @@ namespace Catch {
 
             OriginalExpression = Cyan,
             ReconstructedExpression = BrightYellow,
+            GeneratorValue = Cyan,
 
             SecondaryText = LightGrey,
             Headers = White

diff --git a/src/catch2/internal/catch_generator_info.cpp b/src/catch2/internal/catch_generator_info.cpp
@@ -0,0 +1,20 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/internal/catch_generator_info.hpp>
+
+namespace Catch {
+
+    GeneratorInfo::GeneratorInfo( StringRef _definition,
+                                  SourceLineInfo const& _lineInfo,
+                                  StringRef _currentElement ):
+        definition( _definition ),
+        lineInfo( _lineInfo ),
+        currentElement( _currentElement ) {}
+
+} // end namespace Catch
diff --git a/src/catch2/internal/catch_generator_info.hpp b/src/catch2/internal/catch_generator_info.hpp
@@ -0,0 +1,35 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+#ifndef CATCH_GENERATOR_INFO_HPP_INCLUDED
+#define CATCH_GENERATOR_INFO_HPP_INCLUDED
+
+#include <catch2/interfaces/catch_interfaces_capture.hpp>
+#include <catch2/internal/catch_source_line_info.hpp>
+#include <string>
+
+namespace Catch {
+
+    struct GeneratorInfo {
+        GeneratorInfo( StringRef _definition,
+                       SourceLineInfo const& _lineInfo,
+                       StringRef currentElement );
+
+        StringRef definition;
+        SourceLineInfo lineInfo;
+        StringRef currentElement;
+
+        bool operator==( GeneratorInfo const& other ) const {
+            return definition == other.definition &&
+                   lineInfo == other.lineInfo &&
+                   currentElement == other.currentElement;
+        }
+    };
+
+} // end namespace Catch
+
+#endif // CATCH_GENERATOR_INFO_HPP_INCLUDED
diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp
@@ -283,7 +283,7 @@ namespace Catch {
             m_lastAssertionPassed = true;
         }
 
-        m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals));
+        m_reporter->assertionEnded(AssertionStats(result, m_messages, m_generatorInfos, m_totals));
 
         if (result.getResultType() != ResultWas::Warning)
             m_messageScopes.clear();
@@ -319,6 +319,16 @@ namespace Catch {
         return tracker;
     }
 
+    void RunContext::trackGeneratorState( GeneratorInfo info ) {
+        // Avoid redundant entries, in case a generator is used within a loop.
+        if ( std::find( m_generatorInfos.cbegin(),
+                        m_generatorInfos.cend(),
+                        info ) != m_generatorInfos.cend() )
+            return;
+
+        m_generatorInfos.push_back( info );
+    }
+
     bool RunContext::testForMissingAssertions(Counts& assertions) {
         if (assertions.total() != 0)
             return false;
@@ -343,6 +353,7 @@ namespace Catch {
         m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions));
         m_messages.clear();
         m_messageScopes.clear();
+        m_generatorInfos.clear();
     }
 
     void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) {
@@ -490,6 +501,7 @@ namespace Catch {
         handleUnfinishedSections();
         m_messages.clear();
         m_messageScopes.clear();
+        m_generatorInfos.clear();
 
         SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions);
         m_reporter->sectionEnded(testCaseSectionStats);

diff --git a/src/catch2/internal/catch_run_context.hpp b/src/catch2/internal/catch_run_context.hpp
@@ -8,6 +8,7 @@
 #ifndef CATCH_RUN_CONTEXT_HPP_INCLUDED
 #define CATCH_RUN_CONTEXT_HPP_INCLUDED
 
+#include "catch2/interfaces/catch_interfaces_capture.hpp"
 #include <catch2/interfaces/catch_interfaces_reporter.hpp>
 #include <catch2/internal/catch_test_registry.hpp>
 #include <catch2/internal/catch_fatal_condition_handler.hpp>
@@ -74,6 +75,7 @@ namespace Catch {
         void sectionEndedEarly( SectionEndInfo const& endInfo ) override;
 
         auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
+        void trackGeneratorState( GeneratorInfo info ) override;
 
         void benchmarkPreparing( StringRef name ) override;
         void benchmarkStarting( BenchmarkInfo const& info ) override;
@@ -132,6 +134,7 @@ namespace Catch {
         Totals m_totals;
         IEventListenerPtr m_reporter;
         std::vector<MessageInfo> m_messages;
+        std::vector<GeneratorInfo> m_generatorInfos;
         std::vector<ScopedMessage> m_messageScopes; /* Keeps owners of so-called unscoped messages. */
         AssertionInfo m_lastAssertionInfo;
         std::vector<SectionEndInfo> m_unfinishedSections;

diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp
@@ -126,6 +126,7 @@ class ConsoleAssertionPrinter {
             printResultType();
             printOriginalExpression();
             printReconstructedExpression();
+            printGeneratorStates();
         } else {
             stream << '\n';
         }
@@ -153,6 +154,21 @@ class ConsoleAssertionPrinter {
                    << '\n';
         }
     }
+    void printGeneratorStates() const {
+        if ( stats.generatorInfos.empty() ) {
+            return;
+        }
+
+        stream << "with " << pluralise(stats.generatorInfos.size(), "generator"_sr) << "\n";
+        for ( auto const& info : stats.generatorInfos ) {
+            stream << TextFlow::Column( "line:" ).indent( 2 )
+                   << info.lineInfo.line << ": "
+                   << "GENERATE(" << info.definition << ")\n"
+                   << TextFlow::Column( "value: " ).indent( 2 )
+                   << colourImpl->guardColour( Colour::GeneratorValue )
+                   << info.currentElement << "\n";
+        }
+    }
     void printMessage() const {
         if (!messageLabel.empty())
             stream << messageLabel << ':' << '\n';