feat: Support parsing all describable nodes in gherkin-32 mode (#361)

Historically, we have not supported parsing descriptions as a separate
concept for Examples, Background, Scenario or Scenario Outline. Text
following the keyword line was instead parsed as a multi-line title.

This PR implements support for instead capturing these as standalone
properties when running in cucumber/gherkin compatibility mode.

This borrowed from and builds on work done by @jojo1981 in #254, 
but started from a clean branch as there have been a lot of changes
since that PR was opened.

Fixes #154 #211

Author: acoulton

src/Filter/LineFilter.php

@@ -10,7 +10,6 @@
 
 namespace Behat\Gherkin\Filter;
 
-use Behat\Gherkin\Node\ExampleTableNode;
 use Behat\Gherkin\Node\FeatureNode;
 use Behat\Gherkin\Node\OutlineNode;
 use Behat\Gherkin\Node\ScenarioInterface;
@@ -95,9 +94,7 @@ public function filterFeature(FeatureNode $feature)
                             $filteredTable[$this->filterLine] = $table[$this->filterLine];
                         }
 
-                        $scenario = $scenario->withTables(
-                            [new ExampleTableNode($filteredTable, $exampleTable->getKeyword(), $exampleTable->getTags())],
-                        );
+                        $scenario = $scenario->withTables([$exampleTable->withTable($filteredTable)]);
                     }
                 }
             }

src/Filter/LineRangeFilter.php

@@ -10,7 +10,6 @@
 
 namespace Behat\Gherkin\Filter;
 
-use Behat\Gherkin\Node\ExampleTableNode;
 use Behat\Gherkin\Node\FeatureNode;
 use Behat\Gherkin\Node\OutlineNode;
 use Behat\Gherkin\Node\ScenarioInterface;
@@ -111,7 +110,7 @@ public function filterFeature(FeatureNode $feature)
                     }
 
                     if (count($filteredTable) > 1) {
-                        $exampleTableNodes[] = new ExampleTableNode($filteredTable, $exampleTable->getKeyword(), $exampleTable->getTags());
+                        $exampleTableNodes[] = $exampleTable->withTable($filteredTable);
                     }
                 }
 

src/Filter/NameFilter.php

@@ -10,6 +10,7 @@
 
 namespace Behat\Gherkin\Filter;
 
+use Behat\Gherkin\Node\DescribableNodeInterface;
 use Behat\Gherkin\Node\FeatureNode;
 use Behat\Gherkin\Node\ScenarioInterface;
 
@@ -59,15 +60,25 @@ public function isFeatureMatch(FeatureNode $feature)
      */
     public function isScenarioMatch(ScenarioInterface $scenario)
     {
-        if ($scenario->getTitle() === null) {
+        // Historically (and in legacy GherkinCompatibilityMode), multiline scenario text was all part of the title.
+        // In new GherkinCompatibilityMode the text will be split into a single-line title & multiline description.
+        // For BC, this filter should continue to match on the complete multiline text value.
+        $textParts = array_filter([
+            $scenario->getTitle(),
+            $scenario instanceof DescribableNodeInterface ? $scenario->getDescription() : null,
+        ]);
+
+        if ($textParts === []) {
             return false;
         }
 
-        if ($this->filterString[0] === '/' && preg_match($this->filterString, $scenario->getTitle())) {
+        $textToMatch = implode("\n", $textParts);
+
+        if ($this->filterString[0] === '/' && preg_match($this->filterString, $textToMatch)) {
             return true;
         }
 
-        if (str_contains($scenario->getTitle(), $this->filterString)) {
+        if (str_contains($textToMatch, $this->filterString)) {
             return true;
         }
 

src/GherkinCompatibilityMode.php

@@ -35,14 +35,25 @@ public function shouldRemoveStepKeywordSpace(): bool
     /**
      * @internal
      */
-    public function shouldRemoveFeatureDescriptionPadding(): bool
+    public function shouldRemoveDescriptionPadding(): bool
     {
         return match ($this) {
             self::LEGACY => true,
             default => false,
         };
     }
 
+    /**
+     * @internal
+     */
+    public function allowAllNodeDescriptions(): bool
+    {
+        return match ($this) {
+            self::LEGACY => false,
+            default => true,
+        };
+    }
+
     /**
      * @internal
      */

src/Lexer.php

@@ -80,6 +80,7 @@ class Lexer
     private bool $allowLanguageTag = true;
     private bool $allowFeature = true;
     private bool $allowMultilineArguments = false;
+    private bool $allowExamples = false;
     private bool $allowSteps = false;
     /**
      * @phpstan-var TDocStringSeparator|null
@@ -141,6 +142,7 @@ public function analyse(string $input, string $language = 'en')
         $this->allowFeature = true;
         $this->allowMultilineArguments = false;
         $this->allowSteps = false;
+        $this->allowExamples = false;
 
         if (\func_num_args() > 1) {
             // @codeCoverageIgnoreStart
@@ -584,6 +586,7 @@ protected function scanScenario()
 
         $this->allowMultilineArguments = false;
         $this->allowSteps = true;
+        $this->allowExamples = true;
 
         return $token;
     }
@@ -605,6 +608,7 @@ protected function scanOutline()
 
         $this->allowMultilineArguments = false;
         $this->allowSteps = true;
+        $this->allowExamples = true;
 
         return $token;
     }
@@ -618,6 +622,9 @@ protected function scanOutline()
      */
     protected function scanExamples()
     {
+        if (!$this->allowExamples) {
+            return null;
+        }
         $token = $this->scanTitleLine($this->currentDialect->getExamplesKeywords(), 'Examples');
 
         if ($token === null) {

src/Node/BackgroundNode.php

@@ -17,7 +17,7 @@
  *
  * @final since 4.15.0
  */
-class BackgroundNode implements ScenarioLikeInterface
+class BackgroundNode implements ScenarioLikeInterface, DescribableNodeInterface
 {
     /**
      * @param StepNode[] $steps
@@ -27,6 +27,7 @@ public function __construct(
         private readonly array $steps,
         private readonly string $keyword,
         private readonly int $line,
+        private readonly ?string $description = null,
     ) {
     }
 
@@ -50,6 +51,11 @@ public function getTitle()
         return $this->title;
     }
 
+    public function getDescription(): ?string
+    {
+        return $this->description;
+    }
+
     /**
      * Checks if background has steps.
      *

src/Node/DescribableNodeInterface.php

@@ -0,0 +1,19 @@
+<?php
+
+/*
+ * This file is part of the Behat Gherkin Parser.
+ * (c) Konstantin Kudryashov <ever.zet@gmail.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Behat\Gherkin\Node;
+
+interface DescribableNodeInterface
+{
+    /**
+     * @return ?string
+     */
+    public function getDescription();
+}

src/Node/ExampleTableNode.php

@@ -17,7 +17,7 @@
  *
  * @final since 4.15.0
  */
-class ExampleTableNode extends TableNode implements TaggedNodeInterface
+class ExampleTableNode extends TableNode implements TaggedNodeInterface, DescribableNodeInterface
 {
     use TaggedNodeTrait;
 
@@ -29,6 +29,8 @@ public function __construct(
         array $table,
         private readonly string $keyword,
         private readonly array $tags = [],
+        private readonly ?string $name = null,
+        private readonly ?string $description = null,
     ) {
         parent::__construct($table);
     }
@@ -43,6 +45,16 @@ public function getNodeType()
         return 'ExampleTable';
     }
 
+    public function getName(): ?string
+    {
+        return $this->name;
+    }
+
+    public function getDescription(): ?string
+    {
+        return $this->description;
+    }
+
     public function getTags()
     {
         return $this->tags;
@@ -57,4 +69,18 @@ public function getKeyword()
     {
         return $this->keyword;
     }
+
+    /**
+     * @param array<int, list<string>> $table Table in form of [$rowLineNumber => [$val1, $val2, $val3]]
+     */
+    public function withTable(array $table): self
+    {
+        return new self(
+            $table,
+            $this->keyword,
+            $this->tags,
+            $this->name,
+            $this->description,
+        );
+    }
 }

src/Node/FeatureNode.php

@@ -21,7 +21,7 @@
  *
  * @final since 4.15.0
  */
-class FeatureNode implements KeywordNodeInterface, TaggedNodeInterface
+class FeatureNode implements KeywordNodeInterface, TaggedNodeInterface, DescribableNodeInterface
 {
     use TaggedNodeTrait;
 

src/Node/OutlineNode.php

@@ -17,7 +17,7 @@
  *
  * @final since 4.15.0
  */
-class OutlineNode implements ScenarioInterface
+class OutlineNode implements ScenarioInterface, DescribableNodeInterface
 {
     use TaggedNodeTrait;
 
@@ -42,6 +42,7 @@ public function __construct(
         array|ExampleTableNode $tables,
         private readonly string $keyword,
         private readonly int $line,
+        private readonly ?string $description = null,
     ) {
         $this->tables = is_array($tables) ? $tables : [$tables];
     }
@@ -66,6 +67,11 @@ public function getTitle()
         return $this->title;
     }
 
+    public function getDescription(): ?string
+    {
+        return $this->description;
+    }
+
     public function getTags()
     {
         return $this->tags;
@@ -104,7 +110,7 @@ public function hasExamples()
     /**
      * Builds and returns examples table for the outline.
      *
-     * WARNING: it returns a merged table with tags lost.
+     * WARNING: it returns a merged table with tags, names & descriptions lost.
      *
      * @return ExampleTableNode
      *
@@ -181,6 +187,7 @@ public function withTables(array $exampleTables): self
             $exampleTables,
             $this->keyword,
             $this->line,
+            $this->description,
         );
     }