Consume new CCK Nuget package for Formatter tests; Upgraded dependencies to latest releases of Cucumber assets (#984)

* Upgraded dependencies to latest releases of Cucumber Gherkin 37.0.1, Messages 31.1.0 and HtmlFormatter 22.3.0. This supports the addition of the Location property to Pickles.
Updated Formatters.Tests to use the latest release of the CCK scenarios and samples.

* Update CHANGELOG.md

* undo whitespace changes in Reqnroll.csproj

* add PR number to CHANGELOG

* Update Formatters.Tests samples to CCK v27 release (adds a new All-Statuses scenario and other minor restructuring)

* Updated to Gherkin 38, Messages 32.0.1, and HtmlFormatter 23

* Eliminated (deleted) the CCK assets that will be provided by the CCK nuget package.

* Add CCK package.

* Modified tests to use the new path to the CCK assets.
Adjusted test .csproj to embed the CCK assets in test output assembly.

* Fixes to Formatters test infrastucture for using the new CCK package.
Fixed Formatters to conform to Messages expectation that Pending steps have no error message text.
Moved the 'all-statuses' scenario to NonCompliant until ambiguous step match arguments can be surfaced.

* Fix Formatter.Tests csproj for cross-platform compatibility of embedding the CCK resources.

* Fix test for ambiguous step matching

* Fix CHANGELOG.md

* Upgrade Cucumber.HtmlFormatter to v23.1.0 (and Messages to v32.3.1) to finalize the fix for the issue reported in #1035.
Added a regression test to Formatters.Tests to demonstrate proper UTF-8 handling of attachments (as a non-CCK test).

* Fix a OS-line-ending mistake in the regression test.

* Update Gherkin and Cucumber.CCK package versions

Updated Gherkin to 39.1.0 in Reqnroll.Parser.csproj and Cucumber.CCK to 29.2.2 in Reqnroll.Formatters.Tests.csproj.

---------

Co-authored-by: Gáspár Nagy <gaspar.nagy@gmail.com>

Author: clrudolphi

CHANGELOG.md

@@ -1,6 +1,7 @@
 # [vNext]
 
 ## Improvements:
+* Updated Cucumber dependencies to: Gherkin v39.1.0, Cucumber.Messages v32.0.1 and Cucumber.HtmlFormatter v23.1.0. Formatters.Tests modified by adopting use of Cucumber/CCK (v29.2.2). (#984)
 
 ## Bug fixes:
 

Reqnroll.Parser/Reqnroll.Parser.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Gherkin" Version="35.0.0" />
+    <PackageReference Include="Gherkin" Version="39.1.0" />
   </ItemGroup>
 
   <ItemGroup>

Reqnroll/Formatters/PayloadProcessing/Cucumber/CucumberMessageFactory.cs

@@ -188,7 +188,7 @@ public virtual StepMatchArgument ToStepMatchArgument(TestStepArgument argument)
     {
         return new StepMatchArgument(
             new Group(
-                [],
+                null,
                 argument.StartOffset,
                 argument.Value
             ),
@@ -316,7 +316,7 @@ private static string ToTestStepResultMessage(System.Exception exception, Scenar
         return status switch
         {
             ScenarioExecutionStatus.OK => null,
-            ScenarioExecutionStatus.StepDefinitionPending => exception.Message,
+            ScenarioExecutionStatus.StepDefinitionPending => null,
             ScenarioExecutionStatus.UndefinedStep => exception.Message,
             ScenarioExecutionStatus.BindingError => null,
             ScenarioExecutionStatus.TestError => exception.Message,

Reqnroll/Reqnroll.csproj

@@ -14,17 +14,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference
-      Include="..\Reqnroll.Utils\Reqnroll.Utils.csproj"
-      PackAsComponent="true"
-      PrivateAssets="all" />
+    <ProjectReference Include="..\Reqnroll.Utils\Reqnroll.Utils.csproj" PackAsComponent="true" PrivateAssets="all" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Gherkin" Version="35.0.0" />
+    <PackageReference Include="Gherkin" Version="38.0.0" />
     <PackageReference Include="Cucumber.CucumberExpressions" Version="17.1.0" />
-    <PackageReference Include="Cucumber.Messages" Version="30.1.0" />
-    <PackageReference Include="Cucumber.HtmlFormatter" Version="22.2.0" />
+    <PackageReference Include="Cucumber.Messages" Version="32.3.1" />
+    <PackageReference Include="Cucumber.HtmlFormatter" Version="23.1.0" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.6" />
     <PackageReference Include="Microsoft.Extensions.DependencyModel" Version="8.0.2" />
     <!-- System.Runtime.Loader is used for .NET (not .NET Framework) based frameworks only. 
@@ -41,9 +38,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <EmbeddedResource
-      Include=".\BindingSkeletons\DefaultSkeletonTemplates.sftemplate"
-      Exclude="bin\**;obj\**;**\*.xproj;packages\**;@(EmbeddedResource)" />
+    <EmbeddedResource Include=".\BindingSkeletons\DefaultSkeletonTemplates.sftemplate" Exclude="bin\**;obj\**;**\*.xproj;packages\**;@(EmbeddedResource)" />
   </ItemGroup>
 
   <!-- Ensure the instrumentation key template isn't included as source. -->
@@ -72,23 +67,15 @@
   </ItemGroup>
 
   <!-- Embeds the Application Insights instrumentation key  -->
-  <Target
-    Name="EmbedAppInsightsInstrumentationKey"
-    Inputs="@(AppInsightsInstrumentationKeyFile)"
-    Outputs="$(IntermediateOutputPath)%(AppInsightsInstrumentationKeyFile.Filename).$(AppInsightsInstrumentationKey)%(AppInsightsInstrumentationKeyFile.Extension)"
-    BeforeTargets="CoreCompile">
+  <Target Name="EmbedAppInsightsInstrumentationKey" Inputs="@(AppInsightsInstrumentationKeyFile)" Outputs="$(IntermediateOutputPath)%(AppInsightsInstrumentationKeyFile.Filename).$(AppInsightsInstrumentationKey)%(AppInsightsInstrumentationKeyFile.Extension)" BeforeTargets="CoreCompile">
 
     <!-- Read the lines from the template. -->
     <ReadLinesFromFile File="%(AppInsightsInstrumentationKeyFile.Identity)">
       <Output TaskParameter="Lines" ItemName="_AppInsightsInstrumentationKeyFileTemplateLine" />
     </ReadLinesFromFile>
 
     <!-- Produce the output file with the key token replaced. -->
-    <WriteLinesToFile
-      File="$(IntermediateOutputPath)%(AppInsightsInstrumentationKeyFile.Filename).$(AppInsightsInstrumentationKey)%(AppInsightsInstrumentationKeyFile.Extension)"
-      Lines="@(_AppInsightsInstrumentationKeyFileTemplateLine -> Replace('_InstrumentationKey_', '$(AppInsightsInstrumentationKey)'))"
-      Overwrite="true"
-      Encoding="UTF-8" />
+    <WriteLinesToFile File="$(IntermediateOutputPath)%(AppInsightsInstrumentationKeyFile.Filename).$(AppInsightsInstrumentationKey)%(AppInsightsInstrumentationKeyFile.Extension)" Lines="@(_AppInsightsInstrumentationKeyFileTemplateLine -&gt; Replace('_InstrumentationKey_', '$(AppInsightsInstrumentationKey)'))" Overwrite="true" Encoding="UTF-8" />
 
     <ItemGroup>
       <!-- Include the file in the final build package -->

Tests/Reqnroll.Formatters.Tests/CucumberMessagesBasicTests.cs

@@ -285,7 +285,7 @@ public void ExternalAssembliesAreSupportedInStepDefinitionMessages()
         CucumberMessagesAddConfigurationFile("reqnrollConfigurationWithExternalAssembly.json");
 
         //Set up the Default Project (main test assembly)
-        AddUtilClassWithFileSystemPath();
+        AddUtilClassWithFileSystemPath("Samples");
 
         AddFeatureFileFromResource($"ExternalBindingAssemblies/{featureNameText}.feature", "Samples", Assembly.GetExecutingAssembly());
         AddBindingClassFromResource($"ExternalBindingAssemblies/SampleInternalBindingClass.cs", "Samples", Assembly.GetExecutingAssembly());

Tests/Reqnroll.Formatters.Tests/MessagesCompatibilityTestBase.cs

@@ -201,9 +201,9 @@ protected void FileShouldExist(string v)
         File.Exists(file).Should().BeTrue(file, $"File {v} should exist");
     }
 
-    protected void AddUtilClassWithFileSystemPath()
+    protected void AddUtilClassWithFileSystemPath(string pathsegment)
     {
-        string location = Path.Combine(AppContext.BaseDirectory, "Samples", "Resources");
+        string location = Path.Combine(AppContext.BaseDirectory, pathsegment, "Resources");
         AddBindingClass($$"""
                           public class FileSystemPath 
                           { 
@@ -212,9 +212,9 @@ public class FileSystemPath
                           """);
     }
 
-    protected IEnumerable<Envelope> GetExpectedResults(string testName)
+    protected IEnumerable<Envelope> GetExpectedResults(string testName, string source = "CCK")
     {
-        string[] expectedJsonText = GetExpectedJsonText(testName);
+        string[] expectedJsonText = GetExpectedJsonText(testName, source);
 
         foreach (var json in expectedJsonText)
         {
@@ -223,11 +223,11 @@ protected IEnumerable<Envelope> GetExpectedResults(string testName)
         }
     }
 
-    protected string[] GetExpectedJsonText(string testName)
+    protected string[] GetExpectedJsonText(string testName, string source)
     {
         var fileName = testName + "." + testName + ".ndjson";
         var assemblyToLoadFrom = Assembly.GetExecutingAssembly();
-        var expectedJsonText = _testFileManager.GetTestFileContent(fileName, "Samples", assemblyToLoadFrom).Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
+        var expectedJsonText = _testFileManager.GetTestFileContent(fileName, source, assemblyToLoadFrom).Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
         return expectedJsonText;
     }
 

Tests/Reqnroll.Formatters.Tests/MessagesCompatibilityTests.cs

@@ -7,6 +7,7 @@ namespace Reqnroll.Formatters.Tests;
 public class MessagesCompatibilityTests : MessagesCompatibilityTestBase
 {
     [TestMethod]
+    [DataRow("all-statuses")] 
     [DataRow("ambiguous")]
     [DataRow("backgrounds")]
     [DataRow("cdata")]
@@ -38,7 +39,7 @@ public class MessagesCompatibilityTests : MessagesCompatibilityTestBase
     // that are not material to the CCK spec (such as IDs don't have to be generated in the same order, timestamps don't have to match, etc.)
     // The rules for what must match and what is allowed to not match are built in to a series of custom FluentAssertion validation rules
     // (located in the CucumberMessagesValidator class)
-    public void CCKScenarios(string testName)
+    public void CCKScenarios(string testName, string featureFileSource = "CCK")
     { 
         var featureFileName = testName.Replace("-", "_");
         ResetCucumberMessages(featureFileName);
@@ -48,14 +49,14 @@ public void CCKScenarios(string testName)
 
         CucumberMessagesAddConfigurationFile("reqnroll_withBothFormatters.json");
         MimicAzurePipelinesEnvironment();
-        AddUtilClassWithFileSystemPath();
+        AddUtilClassWithFileSystemPath(featureFileSource);
 
-        AddFeatureFilesFromResources(featureFileName, "Samples", Assembly.GetExecutingAssembly());
+        AddFeatureFilesFromResources(featureFileName, featureFileSource, Assembly.GetExecutingAssembly());
         AddBindingClassFromResource($"{featureFileName}/{featureFileName}.cs", "Samples", Assembly.GetExecutingAssembly());
 
         ExecuteTests();
         var actualResults = GetActualResults(featureFileName).ToArray();
-        var validator = new CucumberMessagesValidator(actualResults, GetExpectedResults(featureFileName).ToArray());
+        var validator = new CucumberMessagesValidator(actualResults, GetExpectedResults(featureFileName, featureFileSource).ToArray());
         validator.ShouldPassBasicStructuralChecks();
         validator.ResultShouldPassAllComparisonTests();
         validator.ResultShouldPassSanityChecks();
@@ -80,13 +81,13 @@ public void CCKScenarios(string testName)
         ConfirmAllTestsRan(numOfTests);
     }
 
-    [Ignore]
     [TestMethod]
+    [DataRow("Regression-RNR1035-NonASCIIcharactersinUTF8Attachment")] 
     // These tests are not (yet) within the CCK but are included here to round out the testing. The expected results were generated by the CucumberMessages plugin.
     // Once the CCK includes these scenarios, the expected results should come from the CCK repo.
     public void NonCCKScenarios(string testName)
     {
-        CCKScenarios(testName);
+        CCKScenarios(testName, "Samples");
     }
 
     [Ignore]
@@ -95,9 +96,11 @@ public void NonCCKScenarios(string testName)
     [DataRow("global-hooks-beforeall-error")] // Cucumber expects execution to continue after a hook failure
     [DataRow("global-hooks-afterall-error")] // Cucumber expects execution to continue after a hook failure
     [DataRow("global-hooks-attachments")] // Fails b/c we cannot obtain Reqnroll OutputHandler in global hooks
+    [DataRow("hooks-skipped")] // Reqnroll terminates execution on first hook failure; Cucumber expects execution of other hooks continue after a hook failure
     [DataRow("unknown-parameter-type")] // Reqnroll does not provide skeletons for missing Parameter Types
     [DataRow("regular-expression")] // Reqnroll does not support optional binding method arguments
     [DataRow("multiple-features-reversed")] // Reqnroll does not have the concept of specifying execution order via run-time parameter
+    [DataRow("test-run-exception")] // Reqnroll does not support a means of simulating a start-up failure
     // These scenarios are from the CCK, but Reqnroll cannot provide a compliant implementation. This is usually the result of differences in behavior or support of Gherkin features.
     // When these scenarios are run, expect them to fail.
     public void NonCompliantCCKScenarios(string testName)
@@ -128,6 +131,6 @@ public void CCKRetryScenario(string pluginName, string testNameRoot)
         _testRunConfiguration.UnitTestProvider = unitTestProvider;
         _projectsDriver.AddNuGetPackage(plugin, version);
         var testName = $"{pluginName}-{testNameRoot}";
-        CCKScenarios(testName);
+        CCKScenarios(testName, "Samples");
     }
 }

Tests/Reqnroll.Formatters.Tests/Reqnroll.Formatters.Tests.csproj

@@ -8,31 +8,42 @@
     <IsPackable>false</IsPackable>
     <IsTestProject>true</IsTestProject>
   </PropertyGroup>
-
+  
+  <!-- This property is used by Cucumber.CCK as destination of the content files (e.g. .feature, .ndjson) for the tests. -->
+  <PropertyGroup>
+    <CucumberCCKContentPath>CCK/Resources</CucumberCCKContentPath>
+    <EmbeddedCCKContentPathPrefix>$(CucumberCCKContentPath)/</EmbeddedCCKContentPathPrefix>
+  </PropertyGroup>
+  
+  <!-- Binding classes for scenarios come from the Samples\Resources tree.
+        This tree also includes feature files and other assets for non-CCK scenarios. -->
   <ItemGroup>
     <Compile Remove="Samples\Resources\**\*.cs" />
-    <None Remove="Samples\Resources\**\*.feature" />
-    <None Remove="Samples\Resources\**\*.feature.ndjson" />
-    <None Remove="Samples\Resources\**\*.ts" />
-    <None Update="Samples\Resources\**\*.jpeg">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Samples\Resources\**\*.png">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Samples\Resources\**\*.pdf">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <None Update="Samples\Resources\**\*.svg">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-    </None>
-    <EmbeddedResource Include="Samples\Resources\**\*.cs" />
-    <EmbeddedResource Include="Samples\Resources\**\*.feature" />
-    <EmbeddedResource Include="Samples\Resources\**\*.ndjson" />
-    <EmbeddedResource Include="Samples\Resources\**\*.csv" />
+    <EmbeddedResource Include="Samples\Resources\**\*.*" />
   </ItemGroup>
 
+  <!-- This target embeds the feature files and other assets delivered by the Cucumber.CCK package. -->
+  <Target Name="MakeCCKContentEmbeddedResource" AfterTargets="ResolvePackageAssets" BeforeTargets="CoreCompile">
+    <ItemGroup>
+      <!-- Step 1: Start with Content items that have Link metadata -->
+      <_CCKContent Include="@(Content)" Condition="'%(Link)' != ''">
+        <LinkNormalized>$([System.String]::Copy('%(Link)').Replace('\','/'))</LinkNormalized>
+      </_CCKContent>
+
+      <!-- Step 2: Filter to only those matching the prefix -->
+      <_CCKContent Update="@(_CCKContent)" Condition="$([System.String]::new('%(LinkNormalized)').StartsWith('$(EmbeddedCCKContentPathPrefix)'))" />
+
+      <!-- Step 3: Remove from Content and compute LogicalName -->
+      <Content Remove="@(_CCKContent)" />
+      <_CCKContent Update="@(_CCKContent)">
+        <LogicalName>$(RootNamespace).$([System.String]::Copy('%(Link)').Replace('\','.').Replace('/','.').Replace('-','_'))</LogicalName>
+      </_CCKContent>
+      <EmbeddedResource Include="@(_CCKContent)" />
+    </ItemGroup>
+  </Target>
+
   <ItemGroup>
+    <PackageReference Include="Cucumber.CCK" Version="29.2.2" />
     <PackageReference Include="GitHubActionsTestLogger" Version="2.4.1" Condition="'$(REQNROLL_TEST_PIPELINEMODE)' == 'true'">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
@@ -45,8 +56,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference
-      Include="..\..\Plugins\Reqnroll.ExternalData\Reqnroll.ExternalData.ReqnrollPlugin\Reqnroll.ExternalData.ReqnrollPlugin.csproj">
+    <ProjectReference Include="..\..\Plugins\Reqnroll.ExternalData\Reqnroll.ExternalData.ReqnrollPlugin\Reqnroll.ExternalData.ReqnrollPlugin.csproj">
       <PrivateAssets>all</PrivateAssets>
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>

Tests/Reqnroll.Formatters.Tests/Samples/Resources/NUnitRetry-retry-pending/NUnitRetry_retry_pending.ndjson

@@ -7,6 +7,6 @@
 {"testCase":{"id":"6","pickleId":"3","testSteps":[{"id":"7","pickleStepId":"2","stepDefinitionIds":["4"],"stepMatchArgumentsLists":[{"stepMatchArguments":[]}]}],"testRunStartedId":"5"}}
 {"testCaseStarted":{"id":"8","testCaseId":"6","timestamp":{"seconds":0,"nanos":1000000},"attempt":0}}
 {"testStepStarted":{"testCaseStartedId":"8","testStepId":"7","timestamp":{"seconds":0,"nanos":2000000}}}
-{"testStepFinished":{"testCaseStartedId":"8","testStepId":"7","testStepResult":{"status":"PENDING","message":"TODO","duration":{"seconds":0,"nanos":1000000}},"timestamp":{"seconds":0,"nanos":3000000}}}
+{"testStepFinished":{"testCaseStartedId":"8","testStepId":"7","testStepResult":{"status":"PENDING","duration":{"seconds":0,"nanos":1000000}},"timestamp":{"seconds":0,"nanos":3000000}}}
 {"testCaseFinished":{"testCaseStartedId":"8","timestamp":{"seconds":0,"nanos":4000000},"willBeRetried":false}}
 {"testRunFinished":{"testRunStartedId":"5","timestamp":{"seconds":0,"nanos":5000000},"success":false}}

Tests/Reqnroll.Formatters.Tests/Samples/Resources/Regression-RNR1035-NonASCIIcharactersinUTF8Attachment/Regression_RNR1035_NonASCIIcharactersinUTF8Attachment.cs

@@ -0,0 +1,33 @@
+using Reqnroll;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CucumberMessages.CompatibilityTests.CCK.RegressionRNR1035NonASCIIcharactersinUTF8Attachment;
+
+[Binding]
+internal class Attachments
+{
+    private readonly IReqnrollOutputHelper reqnrollOutputHelper;
+
+    internal Attachments(IReqnrollOutputHelper reqnrollOutputHelper)
+    {
+        this.reqnrollOutputHelper = reqnrollOutputHelper;
+    }
+
+    [When(@"attaching the non-ASCII string")]
+    public void WhenAttachTextAs(string text)
+    {
+        // Normalize line endings to LF so the file content — and therefore
+        // its Base64 encoding — is identical on Windows and Linux.
+        var normalizedText = text.Replace("\r\n", "\n");
+
+        // write the string to a file as UTF-8 in current directory
+        var fileName = $"Regression-RNR1035-NonASCIIcharactersinUTF8Attachment.txt";
+        System.IO.File.WriteAllText(fileName, normalizedText, Encoding.UTF8);
+        reqnrollOutputHelper.AddAttachment(fileName);
+    }
+
+}