Handle tests marked as inconclusive (#2405)

src/Main.ps1

@@ -747,12 +747,12 @@ function Invoke-Pester {
             }
 
             $plugins.Add((
-                # decorator plugin needs to be added after output
-                # because on teardown they will run in opposite order
-                # and that way output can consume the fixed object that decorator
-                # decorated, not nice but works
-                Get-RSpecObjectDecoratorPlugin
-            ))
+                    # decorator plugin needs to be added after output
+                    # because on teardown they will run in opposite order
+                    # and that way output can consume the fixed object that decorator
+                    # decorated, not nice but works
+                    Get-RSpecObjectDecoratorPlugin
+                ))
 
             if ($PesterPreference.TestDrive.Enabled.Value) {
                 $plugins.Add((Get-TestDrivePlugin))
@@ -1458,11 +1458,13 @@ function ConvertTo-Pester4Result {
                 "Skipped" {
                     $legacyResult.SkippedCount++
                 }
+                "Inconclusive" {
+                    $legacyResult.InconclusiveCount++
+                }
             }
         }
         $legacyResult.TotalCount = $legacyResult.TestResult.Count
         $legacyResult.PendingCount = 0
-        $legacyResult.InconclusiveCount = 0
         $legacyResult.Time = $PesterResult.Duration
 
         $legacyResult

src/Pester.RSpec.ps1

@@ -107,6 +107,9 @@ function Add-RSpecTestObjectProperties {
     $TestObject.Result = if ($TestObject.Skipped) {
         "Skipped"
     }
+    elseif ($TestObject.Inconclusive) {
+        "Inconclusive"
+    }
     elseif ($TestObject.Passed) {
         "Passed"
     }
@@ -160,6 +163,9 @@ function PostProcess-RspecTestRun ($TestRun) {
             "Skipped" {
                 $null = $TestRun.Skipped.Add($t)
             }
+            "Inconclusive" {
+                $null = $TestRun.Inconclusive.Add($t)
+            }
             default { throw "Result $($t.Result) is not supported." }
         }
 
@@ -236,6 +242,7 @@ function PostProcess-RspecTestRun ($TestRun) {
     $TestRun.PassedCount = $TestRun.Passed.Count
     $TestRun.FailedCount = $TestRun.Failed.Count
     $TestRun.SkippedCount = $TestRun.Skipped.Count
+    $TestRun.InconclusiveCount = $TestRun.Inconclusive.Count
     $TestRun.NotRunCount = $TestRun.NotRun.Count
 
     $TestRun.TotalCount = $TestRun.Tests.Count

src/Pester.Runtime.ps1

@@ -687,7 +687,18 @@ function Invoke-TestItem {
                         Write-PesterDebugMessage -Scope Skip "($path) Test is skipped."
                     }
                     $Test.Passed = $true
-                    $Test.Skipped = $true
+                    if ('PesterTestInconclusive' -eq $Result.ErrorRecord.FullyQualifiedErrorId) {
+                        $Test.Inconclusive = $true
+                    }
+                    else {
+                        $Test.Skipped = $true
+
+                        # Pending test is still considered a skipped, we don't have a special category for it.
+                        # Mark the run to show deprecation message.
+                        if ('PesterTestPending' -eq $Result.ErrorRecord.FullyQualifiedErrorId) {
+                            $test.Block.Root.FrameworkData['ShowPendingDeprecation'] = $true
+                        }
+                    }
                 }
                 else {
                     $Test.Passed = $result.Success
@@ -2194,6 +2205,7 @@ function PostProcess-ExecutedBlock {
             $b.OwnFailedCount = 0
             $b.OwnPassedCount = 0
             $b.OwnSkippedCount = 0
+            $b.OwnInconclusiveCount = 0
             $b.OwnNotRunCount = 0
 
             $testDuration = [TimeSpan]::Zero
@@ -2205,6 +2217,9 @@ function PostProcess-ExecutedBlock {
                 if (-not $t.ShouldRun) {
                     $b.OwnNotRunCount++
                 }
+                elseif ($t.ShouldRun -and $t.Inconclusive) {
+                    $b.OwnInconclusiveCount++
+                }
                 elseif ($t.ShouldRun -and $t.Skipped) {
                     $b.OwnSkippedCount++
                 }
@@ -2245,6 +2260,7 @@ function PostProcess-ExecutedBlock {
                 $b.FailedCount = $b.OwnFailedCount
                 $b.PassedCount = $b.OwnPassedCount
                 $b.SkippedCount = $b.OwnSkippedCount
+                $b.InconclusiveCount = $b.OwnInconclusiveCount
                 $b.NotRunCount = $b.OwnNotRunCount
             }
             else {
@@ -2266,6 +2282,7 @@ function PostProcess-ExecutedBlock {
                     $b.PassedCount += $child.PassedCount
                     $b.FailedCount += $child.FailedCount
                     $b.SkippedCount += $child.SkippedCount
+                    $b.InconclusiveCount += $child.InconclusiveCount
                     $b.NotRunCount += $child.NotRunCount
                 }
 
@@ -2274,6 +2291,7 @@ function PostProcess-ExecutedBlock {
                 $b.PassedCount += $b.OwnPassedCount
                 $b.FailedCount += $b.OwnFailedCount
                 $b.SkippedCount += $b.OwnSkippedCount
+                $b.InconclusiveCount += $b.OwnInconclusiveCount
                 $b.NotRunCount += $b.OwnNotRunCount
 
                 $b.Passed = -not ($thisBlockFailed -or $anyTestFailed -or $anyChildBlockFailed)

src/csharp/Pester/Container.cs

@@ -18,6 +18,7 @@ public class Container
                 FailedCount = block.FailedCount,
                 PassedCount = block.PassedCount,
                 SkippedCount = block.SkippedCount,
+                InconclusiveCount = block.InconclusiveCount,
                 NotRunCount = block.NotRunCount,
                 TotalCount = block.TotalCount,
                 ErrorRecord = block.ErrorRecord ?? new List<object>(),
@@ -56,6 +57,7 @@ public static Container CreateFromFile(FileInfo file)
         public int FailedCount { get; set; }
         public int PassedCount { get; set; }
         public int SkippedCount { get; set; }
+        public int InconclusiveCount { get; set; }
         public int NotRunCount { get; set; }
         public int TotalCount { get; set; }
         public List<object> ErrorRecord { get; set; } = new List<object>();

src/csharp/Pester/Run.cs

@@ -20,6 +20,7 @@ public static Run Create()
 
         public int PassedCount { get; set; }
         public int SkippedCount { get; set; }
+        public int InconclusiveCount { get; set; }
         public int NotRunCount { get; set; }
         public int TotalCount { get; set; }
 
@@ -47,6 +48,7 @@ public static Run Create()
 
         public List<Test> Passed { get; set; } = new List<Test>();
         public List<Test> Skipped { get; set; } = new List<Test>();
+        public List<Test> Inconclusive { get; set; } = new List<Test>();
         public List<Test> NotRun { get; set; } = new List<Test>();
         public List<Test> Tests { get; set; } = new List<Test>();
 

src/csharp/Pester/Test.cs

@@ -63,6 +63,7 @@ public Test()
         public DateTime? ExecutedAt { get; set; }
         public bool Passed { get; set; }
         public bool Skipped { get; set; }
+        public bool Inconclusive { get; set; }
 
         public TimeSpan UserDuration { get; set; }
         public TimeSpan FrameworkDuration { get; set; }

src/csharp/Pester/ToStringConverter.cs

@@ -12,6 +12,7 @@ static string ResultToString(string result)
                 "Passed" => "[+]",
                 "Failed" => "[-]",
                 "Skipped" => "[!]",
+                "Inconclusive" => "[?]",
                 "NotRun" => "[ ]",
                 _ => "[ERR]",
             };

src/en-US/about_Pester.help.txt

@@ -255,7 +255,7 @@ LONG DESCRIPTION
     The output ends with a summary of the test results.
 
          Tests completed in 3.47s
-         Passed: 20 Failed: 1 Skipped: 0 Pending: 0 Inconclusive: 0
+         Tests Passed: 20, Failed: 1, Skipped: 0, Inconclusive: 0, NotRun: 0
 
     However, because Pester uses Write-Host, it does not write to the output
     stream (stdout), so there are no output objects to save in a variable or

src/functions/It.ps1

@@ -15,7 +15,7 @@
     In addition to using your own logic to test expectations and throw exceptions,
     you may also use Pester's Should command to perform assertions in plain language.
 
-    You can intentionally mark It block result as inconclusive by using Set-TestInconclusive
+    You can intentionally mark It block result as inconclusive by using Set-ItResult -Inconclusive
     command as the first tested statement in the It block.
 
     .PARAMETER Name

src/functions/Output.ps1

@@ -26,7 +26,7 @@ $script:ReportStrings = DATA {
 
         TestsPassed       = 'Tests Passed: {0}, '
         TestsFailed       = 'Failed: {0}, '
-        TestsSkipped      = 'Skipped: {0} '
+        TestsSkipped      = 'Skipped: {0}, '
         TestsPending      = 'Pending: {0}, '
         TestsInconclusive = 'Inconclusive: {0}, '
         TestsNotRun       = 'NotRun: {0}'
@@ -61,6 +61,7 @@ $script:ReportTheme = DATA {
         Discovery        = 'Magenta'
         Container        = 'Magenta'
         BlockFail        = 'Red'
+        Warning          = 'Yellow'
     }
 }
 
@@ -345,12 +346,12 @@ function Write-PesterReport {
     # else {
     #     $ReportTheme.Information
     # }
-    # $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) {
-    #     $ReportTheme.Inconclusive
-    # }
-    # else {
-    #     $ReportTheme.Information
-    # }
+    $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) {
+        $ReportTheme.Inconclusive
+    }
+    else {
+        $ReportTheme.Information
+    }
 
     # Try {
     #     $PesterStatePassedScenariosCount = $PesterState.PassedScenarios.Count
@@ -374,6 +375,7 @@ function Write-PesterReport {
     Write-PesterHostMessage ($ReportStrings.TestsPassed -f $RunResult.PassedCount) -Foreground $Success -NoNewLine
     Write-PesterHostMessage ($ReportStrings.TestsFailed -f $RunResult.FailedCount) -Foreground $Failure -NoNewLine
     Write-PesterHostMessage ($ReportStrings.TestsSkipped -f $RunResult.SkippedCount) -Foreground $Skipped -NoNewLine
+    Write-PesterHostMessage ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive -NoNewLine
     Write-PesterHostMessage ($ReportStrings.TestsTotal -f $RunResult.TotalCount) -Foreground $Total -NoNewLine
     Write-PesterHostMessage ($ReportStrings.TestsNotRun -f $RunResult.NotRunCount) -Foreground $NotRun
 
@@ -402,6 +404,15 @@ function Write-PesterReport {
     # & $SafeCommands['Write-Host'] ($ReportStrings.TestsPending -f $RunResult.PendingCount) -Foreground $Pending -NoNewLine
     # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive
     # }
+
+    $rootFrameworkData = @($RunResult.Containers.Blocks.Root.FrameworkData)
+    foreach ($frameworkData in $rootFrameworkData) {
+        if ($null -ne $frameworkData -and $frameworkData['ShowPendingDeprecation']) {
+            Write-PesterHostMessage '**DEPRECATED**: The -Pending parameter of Set-ItResult is deprecated. The parameter will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning
+            # Show it only once.
+            break
+        }
+    }
 }
 
 function Write-CoverageReport {
@@ -843,7 +854,7 @@ function Get-WriteScreenPlugin ($Verbosity) {
                 if ($PesterPreference.Output.Verbosity.Value -in 'Detailed', 'Diagnostic') {
                     $because = if ($_test.FailureMessage) { ", because $($_test.FailureMessage)" } else { $null }
                     Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$margin[?] $out" -NoNewLine
-                    Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive ", is inconclusive$because" -NoNewLine
+                    Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$because" -NoNewLine
                     Write-PesterHostMessage -ForegroundColor $ReportTheme.InconclusiveTime " $humanTime"
                 }
 

src/functions/Set-ItResult.ps1

@@ -14,7 +14,7 @@
     backwards compatible
 
     .PARAMETER Inconclusive
-    **DEPRECATED** Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped
+    Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped
 
     .PARAMETER Pending
     **DEPRECATED** Sets the test result to pending. Cannot be used at the same time as -Inconclusive or -Skipped
@@ -29,6 +29,9 @@
     .EXAMPLE
     ```powershell
     Describe "Example" {
+        It "Inconclusive test" {
+            Set-ItResult -Inconclusive -Because "we want it to be inconclusive"
+        }
         It "Skipped test" {
             Set-ItResult -Skipped -Because "we want it to be skipped"
         }
@@ -38,9 +41,11 @@
     the output should be
 
     ```
-    [!] Skipped test is skipped, because we want it to be skipped
-    Tests completed in 0ms
-    Tests Passed: 0, Failed: 0, Skipped: 0, Pending: 0, Inconclusive 1
+    Describing Example
+      [?] Inconclusive test is inconclusive, because we want it to be inconclusive 35ms (32ms|3ms)
+      [!] Skipped test is skipped, because we want it to be skipped 3ms (2ms|1ms)
+    Tests completed in 78ms
+    Tests Passed: 0, Failed: 0, Skipped: 1, Inconclusive: 1, NotRun: 0
     ```
 
     .LINK
@@ -58,14 +63,6 @@
 
     $result = $PSCmdlet.ParameterSetName
 
-    [String]$Message = "is skipped"
-    if ($Result -ne 'Skipped') {
-        [String]$Because = if ($Because) { $Result.ToUpper(), $Because -join ': ' } else { $Result.ToUpper() }
-    }
-    if ($Because) {
-        [String]$Message += ", because $Because"
-    }
-
     switch ($null) {
         $File {
             [String]$File = $MyInvocation.ScriptName
@@ -81,15 +78,25 @@
     switch ($result) {
         'Inconclusive' {
             [String]$errorId = 'PesterTestInconclusive'
+            [String]$message = "is inconclusive"
+            break
         }
         'Pending' {
             [String]$errorId = 'PesterTestPending'
+            [String]$message = "is pending"
+            break
         }
         'Skipped' {
             [String]$errorId = 'PesterTestSkipped'
+            [String]$message = "is skipped"
+            break
         }
     }
 
+    if ($Because) {
+        [String]$message += ", because $(Format-Because $Because)"
+    }
+
     throw [Pester.Factory]::CreateErrorRecord(
         $errorId, #string errorId
         $Message, #string message

src/functions/TestResults.NUnit25.ps1

@@ -13,7 +13,7 @@
 }
 
 function Write-NUnitTestResultAttributes {
-    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')]
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
     param($Result, [System.Xml.XmlWriter] $XmlWriter)
 
     $XmlWriter.WriteAttributeString('xmlns', 'xsi', $null, 'http://www.w3.org/2001/XMLSchema-instance')
@@ -23,7 +23,7 @@ function Write-NUnitTestResultAttributes {
     $XmlWriter.WriteAttributeString('errors', '0')
     $XmlWriter.WriteAttributeString('failures', $Result.FailedCount)
     $XmlWriter.WriteAttributeString('not-run', $Result.NotRunCount)
-    $XmlWriter.WriteAttributeString('inconclusive', '0') # $Result.PendingCount + $Result.InconclusiveCount) #TODO: reflect inconclusive count once it is added
+    $XmlWriter.WriteAttributeString('inconclusive', $Result.InconclusiveCount)
     $XmlWriter.WriteAttributeString('ignored', '0')
     $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount)
     $XmlWriter.WriteAttributeString('invalid', '0')
@@ -32,7 +32,7 @@ function Write-NUnitTestResultAttributes {
 }
 
 function Write-NUnitTestResultChildNodes {
-    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')]
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
     param($Result, [System.Xml.XmlWriter] $XmlWriter)
 
     Write-NUnitEnvironmentInformation -Result $Result -XmlWriter $XmlWriter
@@ -98,7 +98,7 @@ function Write-NUnitCultureInformation {
 }
 
 function Write-NUnitTestSuiteElements {
-    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')]
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
     param($Node, [System.Xml.XmlWriter] $XmlWriter, [string] $Path)
 
     $suiteInfo = Get-TestSuiteInfo -TestSuite $Node -Path $Path
@@ -249,7 +249,7 @@ function Get-TestSuiteInfo {
 }
 
 function Write-NUnitTestSuiteAttributes {
-    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')]
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
     param($TestSuiteInfo, [string] $TestSuiteType = 'TestFixture', [System.Xml.XmlWriter] $XmlWriter, [string] $Path)
 
     $name = $TestSuiteInfo.Name
@@ -279,7 +279,7 @@ function Write-NUnitTestCaseElement {
 }
 
 function Write-NUnitTestCaseAttributes {
-    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')]
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
     param($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $ParameterizedSuiteName)
 
     $testName = $TestResult.ExpandedPath
@@ -399,5 +399,8 @@ function Get-GroupResult ($InputObject) {
     if ($InputObject.PendingCount -gt 0) {
         return 'Inconclusive'
     }
+    if ($InputObject.InconclusiveCount -gt 0) {
+        return 'Inconclusive'
+    }
     return 'Success'
 }