Use list<T> in ArrayBuffer instead of custom keys (possibly keys duplication fix).

Author: SerafimArts

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        php: [ '7.4', '8.0', '8.1', '8.2', '8.3', '8.4' ]
+        php: [ '8.1', '8.2', '8.3', '8.4' ]
         os: [ ubuntu-latest, macos-latest, windows-latest ]
         stability: [ prefer-lowest, prefer-stable ]
     steps:

composer.json

@@ -21,14 +21,18 @@
         "ext-pcre": "*",
         "ext-json": "*",
         "ext-mbstring": "*",
-        "phplrt/source": "^4.0",
-        "phplrt/lexer-contracts": "^4.0",
-        "phplrt/exception": "^4.0"
+        "phplrt/source": "^3.7",
+        "phplrt/lexer-contracts": "^3.7",
+        "phplrt/exception": "^3.7",
+        "symfony/deprecation-contracts": "^2.5|^3.0"
     },
     "autoload": {
         "psr-4": {
             "Phplrt\\Lexer\\": "src"
-        }
+        },
+        "files": [
+            "src/polyfill.php"
+        ]
     },
     "require-dev": {
         "phpunit/phpunit": "^10.5|^11.0",
@@ -42,16 +46,19 @@
         }
     },
     "provide": {
-        "phplrt/lexer-contracts-implementation": "^4.0"
+        "phplrt/lexer-contracts-implementation": "^3.7"
     },
     "extra": {
         "branch-alias": {
-            "dev-master": "4.x-dev",
-            "dev-main": "4.x-dev"
+            "dev-master": "3.x-dev",
+            "dev-main": "3.x-dev"
         }
     },
     "config": {
-        "sort-packages": true
+        "sort-packages": true,
+        "allow-plugins": {
+            "phpstan/extension-installer": true
+        }
     },
     "minimum-stability": "dev",
     "prefer-stable": true

resources/.deprecations.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace Phplrt\Lexer\Buffer {
+
+    /**
+     * @deprecated since phplrt 3.2 and will be removed in 4.0, use {@see \Phplrt\Buffer\ArrayBuffer} instead.
+     */
+    class ArrayBuffer extends \Phplrt\Buffer\ArrayBuffer
+    {
+    }
+
+    /**
+     * @deprecated since phplrt 3.2 and will be removed in 4.0, use {@see \Phplrt\Buffer\ExtrusiveBuffer} instead.
+     */
+    class ExtrusiveBuffer extends \Phplrt\Buffer\ExtrusiveBuffer
+    {
+    }
+
+    /**
+     * @deprecated since phplrt 3.2 and will be removed in 4.0, use {@see \Phplrt\Buffer\LazyBuffer} instead.
+     */
+    class LazyBuffer extends \Phplrt\Buffer\LazyBuffer
+    {
+    }
+
+    /**
+     * @deprecated since phplrt 3.2 and will be removed in 4.0, use {@see \Phplrt\Buffer\Buffer} instead.
+     */
+    abstract class Buffer extends \Phplrt\Buffer\Buffer
+    {
+    }
+
+}

src/Compiler/Markers.php

@@ -26,9 +26,6 @@ class Markers extends PCRECompiler
      * @param array<array-key, string> $chunks
      *
      * @return non-empty-string
-     *
-     * @psalm-suppress MoreSpecificReturnType
-     * @psalm-suppress LessSpecificReturnStatement
      */
     protected function buildTokens(array $chunks): string
     {

src/Compiler/PCRECompiler.php

@@ -175,7 +175,6 @@ abstract class PCRECompiler implements CompilerInterface
      */
     public function __construct(?array $flags = null, ?bool $debug = null)
     {
-        /** @psalm-suppress PropertyTypeCoercion */
         $this->flags = $flags ?? self::DEFAULT_FLAGS;
 
         if ($debug === null) {
@@ -269,7 +268,7 @@ protected function test(string $pattern, ?string $original = null): void
 
             @\preg_match_all($this->wrap($pattern), '', $matches, $flags);
 
-            if ($error = \error_get_last()) {
+            if (($error = \error_get_last()) !== null) {
                 throw new CompilationException($this->formatException($error['message'], $original));
             }
         }
@@ -291,8 +290,8 @@ protected function formatException(string $message, ?string $token = null): stri
 
         $message = \str_replace('Compilation failed: ', '', $message);
         $message = \preg_replace('/([\w_]+\(\):\h+)/', '', $message);
-        $message = \preg_replace('/\h*at\h+offset\h+\d+/', '', $message);
+        $message = \preg_replace('/\h*at\h+offset\h+\d+/', '', (string) $message);
 
-        return \ucfirst($message) . (\is_string($token) ? $suffix : '');
+        return \ucfirst((string) $message) . (\is_string($token) ? $suffix : '');
     }
 }

src/Driver/Markers.php

@@ -26,7 +26,7 @@ class Markers extends Driver
 
     public function __construct(
         ?MarkersCompiler $compiler = null,
-        private readonly string $unknown = self::UNKNOWN_TOKEN_NAME
+        private readonly string $unknown = self::UNKNOWN_TOKEN_NAME,
     ) {
         parent::__construct($compiler ?? new MarkersCompiler());
     }
@@ -50,7 +50,6 @@ public function run(array $tokens, ReadableInterface $source, int $offset = 0):
             /** @var non-empty-string $name */
             $name = \array_pop($payload);
 
-            /** @psalm-suppress InvalidArgument */
             yield $this->make($name, $payload);
         }
     }

src/Exception/EndlessRecursionException.php

@@ -9,8 +9,12 @@
 
 class EndlessRecursionException extends UnexpectedStateException
 {
-    public static function fromState(mixed $state, ReadableInterface $src, ?TokenInterface $tok, ?\Throwable $e = null): self
-    {
+    public static function fromState(
+        mixed $state,
+        ReadableInterface $src,
+        ?TokenInterface $tok,
+        ?\Throwable $e = null,
+    ): self {
         $message = \vsprintf('An unsolvable infinite lexer state transitions was found at %s', [
             $state,
         ]);

src/Exception/UnexpectedStateException.php

@@ -16,6 +16,9 @@ public static function fromEmptyStates(ReadableInterface $src, ?\Throwable $e =
         return new static($message, $src, null, $e);
     }
 
+    /**
+     * @param array-key $state
+     */
     public static function fromState(string|int $state, ReadableInterface $src, ?TokenInterface $tok, ?\Throwable $e = null): self
     {
         $message = \sprintf('Unrecognized token state #%s', $state);

src/Lexer.php

@@ -31,19 +31,15 @@ class Lexer implements PositionalLexerInterface, MutableLexerInterface
      * Default token name for unidentified tokens.
      *
      * @var non-empty-string
-     *
-     * @final
      */
-    public const DEFAULT_UNKNOWN_TOKEN_NAME = UnknownToken::DEFAULT_TOKEN_NAME;
+    final public const DEFAULT_UNKNOWN_TOKEN_NAME = UnknownToken::DEFAULT_TOKEN_NAME;
 
     /**
      * Default token name for end of input.
      *
      * @var non-empty-string
-     *
-     * @final
      */
-    public const DEFAULT_EOI_TOKEN_NAME = EndOfInput::DEFAULT_TOKEN_NAME;
+    final public const DEFAULT_EOI_TOKEN_NAME = EndOfInput::DEFAULT_TOKEN_NAME;
 
     private DriverInterface $driver;
 
@@ -53,17 +49,9 @@ class Lexer implements PositionalLexerInterface, MutableLexerInterface
 
     private HandlerInterface $onEndOfInput;
 
-    /**
-     * @var non-empty-string
-     */
-    private readonly string $unknown;
-
     private readonly SourceFactoryInterface $sources;
 
     /**
-     * @param array<array-key, non-empty-string> $tokens list of
-     *        token names/identifiers and its patterns
-     * @param list<array-key> $skip list of hidden token names/identifiers
      * @param HandlerInterface $onUnknownToken This setting is responsible for
      *        the behavior of the lexer in case of detection of unrecognized
      *        tokens.
@@ -87,37 +75,61 @@ class Lexer implements PositionalLexerInterface, MutableLexerInterface
      *
      *        Note that you can also define your own {@see HandlerInterface} to
      *        override behavior.
-     * @param non-empty-string $unknown The identifier that marks each unknown
-     *        token inside the executor (internal runtime). This parameter only
-     *        needs to be changed if the name is already in use in the user's
-     *        token set (in the {@see $tokens} parameter), otherwise it makes
-     *        no sense.
-     * @param non-empty-string $eoi
      */
     public function __construct(
+        /**
+         * List of token names/identifiers and its patterns.
+         *
+         * @var array<array-key, non-empty-string>
+         */
         protected array $tokens = [],
+        /**
+         * List of hidden token names/identifiers.
+         *
+         * @var list<array-key>
+         */
         protected array $skip = [],
         ?DriverInterface $driver = null,
         ?HandlerInterface $onHiddenToken = null,
         ?HandlerInterface $onUnknownToken = null,
         ?HandlerInterface $onEndOfInput = null,
-        string $unknown = Lexer::DEFAULT_UNKNOWN_TOKEN_NAME,
         /**
-         * @readonly
+         * The identifier that marks each unknown token inside the executor
+         * (internal runtime). This parameter only needs to be changed if the
+         * name is already in use in the user's token set (in the {@see $tokens}
+         * parameter), otherwise it makes no sense.
+         *
+         * @var non-empty-string
+         */
+        private readonly string $unknown = Lexer::DEFAULT_UNKNOWN_TOKEN_NAME,
+        /**
+         * @var non-empty-string
          */
-        private string $eoi = Lexer::DEFAULT_EOI_TOKEN_NAME,
-        ?SourceFactoryInterface $sources = null
+        private readonly string $eoi = Lexer::DEFAULT_EOI_TOKEN_NAME,
+        ?SourceFactoryInterface $sources = null,
     ) {
         $this->driver = $driver ?? new Markers(new MarkersCompiler(), $unknown);
-        $this->unknown = $unknown;
-
         $this->onHiddenToken = $onHiddenToken ?? new NullHandler();
         $this->onUnknownToken = $onUnknownToken ?? new ThrowErrorHandler();
         $this->onEndOfInput = $onEndOfInput ?? new PassthroughHandler();
-
         $this->sources = $sources ?? new SourceFactory();
     }
 
+    /**
+     * @deprecated since phplrt 3.6 and will be removed in 4.0. Please use
+     *             "$onUnknownToken" argument of the {@see __construct()}
+     *             or {@see Lexer::withUnknownTokenHandler()} method instead.
+     */
+    public function disableUnrecognizedTokenException(): void
+    {
+        trigger_deprecation('phplrt/lexer', '3.6', <<<'MSG'
+            Using "%s::disableUnrecognizedTokenException()" is deprecated.
+            Please use %1$s::withUnknownTokenHandler() instead.
+            MSG, static::class);
+
+        $this->onUnknownToken = new PassthroughHandler();
+    }
+
     /**
      * @psalm-immutable This method returns a new {@see LexerInterface} instance
      *                  and does not change the current state of the lexer.
@@ -169,6 +181,36 @@ public function withEndOfInputHandler(HandlerInterface $handler): self
         return $self;
     }
 
+    /**
+     * @deprecated since phplrt 3.6 and will be removed in 4.0.
+     *
+     * @api
+     */
+    public function getDriver(): DriverInterface
+    {
+        trigger_deprecation('phplrt/lexer', '3.6', <<<'MSG'
+            Using "%s::getDriver()" is deprecated.
+            MSG, static::class);
+
+        return $this->driver;
+    }
+
+    /**
+     * @deprecated since phplrt 3.6 and will be removed in 4.0.
+     *
+     * @api
+     */
+    public function setDriver(DriverInterface $driver): self
+    {
+        trigger_deprecation('phplrt/lexer', '3.6', <<<'MSG'
+            Using "%s::setDriver(DriverInterface $driver)" is deprecated.
+            MSG, static::class);
+
+        $this->driver = $driver;
+
+        return $this;
+    }
+
     public function skip(string ...$tokens): self
     {
         $this->skip = \array_merge($this->skip, $tokens);
@@ -277,9 +319,13 @@ public function lex(mixed $source, int $offset = 0): iterable
                     continue;
                 }
 
-                if ($unknown !== [] && ($result = $this->handleUnknownToken($source, $unknown))) {
-                    yield $result;
-                    $unknown = [];
+                if ($unknown !== []) {
+                    $result = $this->handleUnknownToken($source, $unknown);
+
+                    if ($result !== null) {
+                        yield $result;
+                        $unknown = [];
+                    }
                 }
 
                 yield $token;
@@ -288,11 +334,17 @@ public function lex(mixed $source, int $offset = 0): iterable
             throw LexerException::fromInternalError($e);
         }
 
-        if ($unknown !== [] && $result = $this->handleUnknownToken($source, $unknown)) {
-            yield $token = $result;
+        if ($unknown !== []) {
+            $result = $this->handleUnknownToken($source, $unknown);
+
+            if ($result !== null) {
+                yield $token = $result;
+            }
         }
 
-        if (($eoi = $this->handleEoiToken($source, $token ?? null)) !== null) {
+        $eoi = $this->handleEoiToken($source, $token ?? null);
+
+        if ($eoi !== null) {
             yield $eoi;
         }
     }
@@ -317,9 +369,7 @@ private function handleEoiToken(ReadableInterface $source, ?TokenInterface $last
      */
     private function reduceUnknownToken(array $tokens): TokenInterface
     {
-        $concat = static function (string $data, TokenInterface $token): string {
-            return $data . $token->getValue();
-        };
+        $concat = static fn(string $data, TokenInterface $token): string => $data . $token->getValue();
 
         $value = \array_reduce($tokens, $concat, '');