diff --git a/Classes/Domain/DTO/FallbackChain.php b/Classes/Domain/DTO/FallbackChain.php new file mode 100644 index 000000000..9f9a7c9b3 --- /dev/null +++ b/Classes/Domain/DTO/FallbackChain.php @@ -0,0 +1,164 @@ + $configurationIdentifiers Ordered list of LlmConfiguration identifiers (should be pre-normalised) + */ + public function __construct( + public array $configurationIdentifiers = [], + ) {} + + /** + * @param array{configurationIdentifiers?: mixed} $data + */ + public static function fromArray(array $data): self + { + $identifiers = $data['configurationIdentifiers'] ?? []; + if (!is_array($identifiers)) { + return new self(); + } + return new self(self::sanitize($identifiers)); + } + + public static function fromJson(string $json): self + { + if ($json === '') { + return new self(); + } + $data = json_decode($json, true); + if (!is_array($data)) { + return new self(); + } + /** @var array{configurationIdentifiers?: mixed} $data */ + return self::fromArray($data); + } + + /** + * @return array{configurationIdentifiers: list} + */ + public function toArray(): array + { + return [ + 'configurationIdentifiers' => $this->configurationIdentifiers, + ]; + } + + public function toJson(): string + { + return json_encode($this->toArray(), JSON_THROW_ON_ERROR); + } + + /** + * @return array{configurationIdentifiers: list} + */ + public function jsonSerialize(): array + { + return $this->toArray(); + } + + public function isEmpty(): bool + { + return $this->configurationIdentifiers === []; + } + + public function count(): int + { + return count($this->configurationIdentifiers); + } + + public function contains(string $identifier): bool + { + return in_array(self::normalise($identifier), $this->configurationIdentifiers, true); + } + + /** + * Return a new chain with the given identifier appended, deduplicated. + * Empty / whitespace-only identifiers are silently ignored. + */ + public function withLink(string $identifier): self + { + $normalised = self::normalise($identifier); + if ($normalised === '' || $this->contains($normalised)) { + return $this; + } + return new self([...$this->configurationIdentifiers, $normalised]); + } + + /** + * Return a new chain without the given identifier. + * Useful for excluding the primary configuration before walking fallbacks. + */ + public function without(string $identifier): self + { + $normalised = self::normalise($identifier); + if ($normalised === '' || !$this->contains($normalised)) { + return $this; + } + $filtered = array_values(array_filter( + $this->configurationIdentifiers, + static fn(string $link): bool => $link !== $normalised, + )); + return new self($filtered); + } + + /** + * Normalise input: drop non-strings, drop empty strings, trim, lowercase, + * drop duplicates, reindex as list. Matches the sanitisation TCA already + * applies to `tx_nrllm_configuration.identifier` so round-trips through + * hand-written JSON do not accidentally miss a row. + * + * @param array $identifiers + * + * @return list + */ + private static function sanitize(array $identifiers): array + { + $seen = []; + $out = []; + foreach ($identifiers as $identifier) { + if (!is_string($identifier)) { + continue; + } + $normalised = self::normalise($identifier); + if ($normalised === '' || isset($seen[$normalised])) { + continue; + } + $seen[$normalised] = true; + $out[] = $normalised; + } + return $out; + } + + private static function normalise(string $identifier): string + { + return strtolower(trim($identifier)); + } +} diff --git a/Classes/Domain/Model/LlmConfiguration.php b/Classes/Domain/Model/LlmConfiguration.php index 8a6ab8a8c..f52ed64e0 100644 --- a/Classes/Domain/Model/LlmConfiguration.php +++ b/Classes/Domain/Model/LlmConfiguration.php @@ -9,6 +9,7 @@ namespace Netresearch\NrLlm\Domain\Model; +use Netresearch\NrLlm\Domain\DTO\FallbackChain; use Netresearch\NrLlm\Domain\DTO\ModelSelectionCriteria; use Netresearch\NrLlm\Domain\Enum\ModelSelectionMode; use Netresearch\NrLlm\Service\Option\ChatOptions; @@ -62,6 +63,12 @@ class LlmConfiguration extends AbstractEntity protected int $maxRequestsPerDay = 0; protected int $maxTokensPerDay = 0; protected float $maxCostPerDay = 0.0; + /** + * JSON-encoded fallback chain (ordered list of LlmConfiguration identifiers). + * Populated automatically from the `fallback_chain` column by Extbase. + */ + protected string $fallbackChain = ''; + protected bool $isActive = true; protected bool $isDefault = false; protected int $allowedGroups = 0; @@ -326,6 +333,32 @@ public function getCrdate(): int return $this->crdate; } + /** + * Get the raw JSON fallback chain (as stored in the database). + * + * Extbase reads this getter during property mapping. + */ + public function getFallbackChain(): string + { + return $this->fallbackChain; + } + + /** + * Get fallback chain as typed DTO. + * + * Returns an empty chain when nothing is configured, so callers never + * have to null-check before calling isEmpty() / configurationIdentifiers. + */ + public function getFallbackChainDTO(): FallbackChain + { + return FallbackChain::fromJson($this->fallbackChain); + } + + public function hasFallbackChain(): bool + { + return !$this->getFallbackChainDTO()->isEmpty(); + } + // ======================================== // Setters // ======================================== @@ -466,6 +499,22 @@ public function setAllowedGroups(int $allowedGroups): void $this->allowedGroups = $allowedGroups; } + /** + * Set fallback chain from raw JSON (used by Extbase when hydrating from DB). + */ + public function setFallbackChain(string $fallbackChain): void + { + $this->fallbackChain = $fallbackChain; + } + + /** + * Set fallback chain from typed DTO. + */ + public function setFallbackChainDTO(FallbackChain $fallbackChain): void + { + $this->fallbackChain = $fallbackChain->isEmpty() ? '' : $fallbackChain->toJson(); + } + /** * @param ObjectStorage|null $beGroups */ diff --git a/Classes/Provider/Exception/FallbackChainExhaustedException.php b/Classes/Provider/Exception/FallbackChainExhaustedException.php new file mode 100644 index 000000000..74dbed093 --- /dev/null +++ b/Classes/Provider/Exception/FallbackChainExhaustedException.php @@ -0,0 +1,71 @@ + $attemptErrors + */ + public function __construct( + string $message, + int $code, + private readonly array $attemptErrors, + ?Throwable $previous = null, + ) { + parent::__construct($message, $code, $previous); + } + + /** + * @param list $attempts + */ + public static function fromAttempts(array $attempts): self + { + $configurations = array_map( + static fn(array $attempt): string => $attempt['configuration'], + $attempts, + ); + $last = $attempts === [] ? null : $attempts[array_key_last($attempts)]['error']; + + $message = sprintf( + 'All %d configuration(s) in the fallback chain failed: %s', + count($attempts), + implode(' -> ', $configurations), + ); + + return new self($message, 1745712001, $attempts, $last); + } + + /** + * @return list + */ + public function getAttemptErrors(): array + { + return $this->attemptErrors; + } + + /** + * @return list + */ + public function getAttemptedConfigurations(): array + { + return array_map( + static fn(array $attempt): string => $attempt['configuration'], + $this->attemptErrors, + ); + } +} diff --git a/Classes/Service/FallbackChainExecutor.php b/Classes/Service/FallbackChainExecutor.php new file mode 100644 index 000000000..1b1a6fd80 --- /dev/null +++ b/Classes/Service/FallbackChainExecutor.php @@ -0,0 +1,160 @@ +getFallbackChainDTO()->isEmpty()) { + return $execute($primary); + } + + /** @var list $attempts */ + $attempts = []; + + try { + return $execute($primary); + } catch (Throwable $e) { + if (!$this->isRetryable($e)) { + throw $e; + } + $attempts[] = [ + 'configuration' => $primary->getIdentifier(), + 'error' => $e, + ]; + $this->logger->warning( + 'LLM primary configuration failed, trying fallback chain', + [ + 'configuration' => $primary->getIdentifier(), + 'exception' => $e, + 'exceptionClass' => $e::class, + ], + ); + } + + // Edge case: a chain that contains only the primary identifier becomes + // empty after filtering. No fallback was actually attempted — rethrow + // the primary's error verbatim instead of wrapping one attempt as + // "every configuration failed". + $chain = $primary->getFallbackChainDTO()->without($primary->getIdentifier()); + if ($chain->isEmpty()) { + $this->logger->warning( + 'LLM primary configuration failed; fallback chain contained only the primary, nothing left to try', + [ + 'configuration' => $primary->getIdentifier(), + 'configured_chain' => $primary->getFallbackChainDTO()->configurationIdentifiers, + ], + ); + throw $attempts[0]['error']; + } + + foreach ($chain->configurationIdentifiers as $identifier) { + $fallback = $this->repository->findOneByIdentifier($identifier); + if ($fallback === null) { + $this->logger->warning( + 'LLM fallback configuration not found, skipping', + ['configuration' => $identifier], + ); + continue; + } + if (!$fallback->isActive()) { + $this->logger->warning( + 'LLM fallback configuration is inactive, skipping', + ['configuration' => $identifier], + ); + continue; + } + + try { + $result = $execute($fallback); + $this->logger->info( + 'LLM fallback configuration succeeded', + [ + 'primary' => $primary->getIdentifier(), + 'fallback' => $identifier, + 'skipped_attempts' => count($attempts), + ], + ); + return $result; + } catch (Throwable $e) { + if (!$this->isRetryable($e)) { + throw $e; + } + $attempts[] = [ + 'configuration' => $identifier, + 'error' => $e, + ]; + $this->logger->warning( + 'LLM fallback configuration failed', + [ + 'configuration' => $identifier, + 'exception' => $e, + 'exceptionClass' => $e::class, + ], + ); + } + } + + throw FallbackChainExhaustedException::fromAttempts($attempts); + } + + private function isRetryable(Throwable $e): bool + { + if ($e instanceof ProviderConnectionException) { + return true; + } + if ($e instanceof ProviderResponseException) { + // Rate limit: a different provider might not be throttled + return $e->getCode() === 429; + } + return false; + } +} diff --git a/Classes/Service/LlmServiceManager.php b/Classes/Service/LlmServiceManager.php index 36744ad11..74b0f1a17 100644 --- a/Classes/Service/LlmServiceManager.php +++ b/Classes/Service/LlmServiceManager.php @@ -45,6 +45,7 @@ public function __construct( private readonly ExtensionConfiguration $extensionConfiguration, private readonly LoggerInterface $logger, private readonly ProviderAdapterRegistry $adapterRegistry, + private readonly ?FallbackChainExecutor $fallbackChainExecutor = null, ) { $this->loadConfiguration(); } @@ -335,36 +336,71 @@ public function getAdapterFromConfiguration(LlmConfiguration $configuration): Pr /** * Execute chat completion using an LlmConfiguration entity. * + * If the configuration has a fallback chain, retryable provider errors + * on the primary (network, 5xx, 429) transparently re-run the request + * against each fallback configuration in order. + * * @param array $messages */ public function chatWithConfiguration(array $messages, LlmConfiguration $configuration): CompletionResponse { - $adapter = $this->getAdapterFromConfiguration($configuration); - $options = $configuration->toOptionsArray(); - - // Remove provider key as we already have the adapter - unset($options['provider']); - - return $adapter->chatCompletion($messages, $options); + return $this->runWithFallback( + $configuration, + function (LlmConfiguration $config) use ($messages): CompletionResponse { + $adapter = $this->getAdapterFromConfiguration($config); + $options = $config->toOptionsArray(); + unset($options['provider']); + return $adapter->chatCompletion($messages, $options); + }, + ); } /** * Execute completion using an LlmConfiguration entity. + * + * Fallback chain is applied when configured; see chatWithConfiguration(). */ public function completeWithConfiguration(string $prompt, LlmConfiguration $configuration): CompletionResponse { - $adapter = $this->getAdapterFromConfiguration($configuration); - $options = $configuration->toOptionsArray(); - - // Remove provider key as we already have the adapter - unset($options['provider']); + return $this->runWithFallback( + $configuration, + function (LlmConfiguration $config) use ($prompt): CompletionResponse { + $adapter = $this->getAdapterFromConfiguration($config); + $options = $config->toOptionsArray(); + unset($options['provider']); + return $adapter->complete($prompt, $options); + }, + ); + } - return $adapter->complete($prompt, $options); + /** + * Apply the fallback chain to a per-configuration call. + * + * When no executor was injected (e.g. manual instantiation in tests), + * or when the configuration has no fallback chain, this delegates + * directly to the callable with the original configuration. + * + * @template T + * + * @param callable(LlmConfiguration): T $execute + * + * @return T + */ + private function runWithFallback(LlmConfiguration $configuration, callable $execute): mixed + { + if ($this->fallbackChainExecutor === null || !$configuration->hasFallbackChain()) { + return $execute($configuration); + } + return $this->fallbackChainExecutor->execute($configuration, $execute); } /** * Stream chat completion using an LlmConfiguration entity. * + * Fallback chain is intentionally NOT applied to streaming: once the first + * chunk has been yielded to the caller we cannot swap providers mid-stream. + * Use chatWithConfiguration() if fallback protection is required. + * * @param array $messages * * @return Generator diff --git a/Configuration/TCA/tx_nrllm_configuration.php b/Configuration/TCA/tx_nrllm_configuration.php index aba9fc9cc..298e0808c 100644 --- a/Configuration/TCA/tx_nrllm_configuration.php +++ b/Configuration/TCA/tx_nrllm_configuration.php @@ -39,6 +39,8 @@ options, --div--;LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tab.limits, --palette--;;limits, + --div--;LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tab.fallback, + fallback_chain, --div--;LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tab.access, allowed_groups, --div--;core.form.tabs:access, @@ -336,6 +338,18 @@ 'default' => 0.00, ], ], + 'fallback_chain' => [ + 'label' => 'LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tx_nrllm_configuration.fallback_chain', + 'description' => 'LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tx_nrllm_configuration.fallback_chain.description', + 'config' => [ + 'type' => 'text', + 'cols' => 60, + 'rows' => 4, + 'eval' => 'trim', + 'placeholder' => '{"configurationIdentifiers":["claude-sonnet","ollama-local"]}', + 'default' => '', + ], + ], 'is_active' => [ 'label' => 'LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tx_nrllm_configuration.is_active', 'description' => 'LLL:EXT:nr_llm/Resources/Private/Language/locallang_tca.xlf:tx_nrllm_configuration.is_active.description', diff --git a/Documentation/Adr/Adr021ProviderFallbackChain.rst b/Documentation/Adr/Adr021ProviderFallbackChain.rst new file mode 100644 index 000000000..c4734c0fb --- /dev/null +++ b/Documentation/Adr/Adr021ProviderFallbackChain.rst @@ -0,0 +1,92 @@ +.. include:: /Includes.rst.txt + +.. _adr-021: + +========================================== +ADR-021: Provider Fallback Chain +========================================== + +:Status: Accepted +:Date: 2026-04 +:Authors: Netresearch DTT GmbH + +.. _adr-021-context: + +Context +======= + +A single misbehaving provider (OpenAI rate-limit, Claude outage, local Ollama +daemon not running) previously bubbled up as an uncaught exception to every +consuming extension. Operators had no built-in way to degrade gracefully to a +second or third provider. + +.. _adr-021-decision: + +Decision +======== + +A configuration's ``fallback_chain`` column stores an ordered JSON list of +other :php:`LlmConfiguration` identifiers. On retryable failures during +:php:`LlmServiceManager::chatWithConfiguration()` or +:php:`completeWithConfiguration()`, a :php:`FallbackChainExecutor` walks the +chain and returns the first successful response — or throws +:php:`FallbackChainExhaustedException` carrying every attempt error. + +"Retryable" is narrowly defined: the request *might* succeed against a +different provider. + +* :php:`ProviderConnectionException` — network / timeout / HTTP 5xx / + retries exhausted +* :php:`ProviderResponseException` with HTTP code 429 — this provider is + rate-limiting us, another might not be + +Everything else (authentication, bad request, unsupported feature, +misconfiguration) bubbles up unchanged — a different provider won't help. + +.. _adr-021-scope: + +Scope limitations (v1) +====================== + +* **Streaming is not wrapped.** Once the first chunk has been yielded, we + cannot swap providers mid-stream. :php:`streamChatWithConfiguration()` + calls the primary adapter directly. + +* **Shallow only.** A fallback configuration's own chain is ignored. This + prevents both cycles (``a -> b -> a``) and exponential blow-up of attempts. + +* **Inactive fallbacks are skipped**, not treated as failures. + +* **Missing identifiers are skipped** with a warning log, not treated as + failures. Misconfiguration should not mask outages. + +.. _adr-021-storage: + +Storage +======= + +The chain is stored as a single JSON column to keep the schema change +minimal and avoid an additional relation table. The +:php:`Netresearch\\NrLlm\\Domain\\DTO\\FallbackChain` value object handles +serialization, deduplication, and order preservation. + +TCA presents the field as a JSON textarea for v1. A richer UI (sortable +multi-select of available configurations) can replace the textarea without +schema or API change. + +.. _adr-021-alternatives: + +Alternatives considered +======================= + +* **Fat middleware pipeline** (as in b13/aim). Rejected for this release — + too invasive for a single-feature change. The middleware pattern remains + on the roadmap as a v1.0 refactor; a fallback chain is the most valuable + pipeline step users ask for and works fine as a standalone service. + +* **Recursive chain resolution** (fallback's fallback). Rejected as the + cost (cycle detection, attempt amplification) outweighs the benefit; + operators can always append to the primary's chain directly. + +* **Per-link retry policy** (per fallback: max retries, backoff, which + exceptions). Rejected as over-engineered for the initial release. diff --git a/Documentation/Adr/Index.rst b/Documentation/Adr/Index.rst index e4af3c252..1b072507c 100644 --- a/Documentation/Adr/Index.rst +++ b/Documentation/Adr/Index.rst @@ -253,3 +253,4 @@ Modern architecture (v0.4+) Adr018MultiProviderModelDiscovery Adr019InternationalizationStrategy Adr020BackendOutputFormatRendering + Adr021ProviderFallbackChain diff --git a/Resources/Private/Language/de.locallang_tca.xlf b/Resources/Private/Language/de.locallang_tca.xlf index 0c4e577bd..aee8bed54 100644 --- a/Resources/Private/Language/de.locallang_tca.xlf +++ b/Resources/Private/Language/de.locallang_tca.xlf @@ -21,6 +21,10 @@ Access Control Zugriffskontrolle + + Fallback Chain + Fallback-Kette + @@ -190,6 +194,15 @@ Tägliches Kostenlimit in Dollar (0,00 = unbegrenzt) + + Fallback Chain + Fallback-Kette + + + Ordered list of configuration identifiers to retry against when the primary fails with a connection error, 5xx, or 429. JSON format: {"configurationIdentifiers":["claude-sonnet","ollama-local"]}. Leave empty to disable fallback. Does not apply to streaming requests. + Geordnete Liste von Konfigurations-Bezeichnern, die bei Verbindungsfehlern, 5xx oder 429 nacheinander versucht werden. JSON-Format: {"configurationIdentifiers":["claude-sonnet","ollama-local"]}. Leer lassen, um Fallback zu deaktivieren. Gilt nicht für Streaming-Anfragen. + + Active Aktiv diff --git a/Resources/Private/Language/locallang_tca.xlf b/Resources/Private/Language/locallang_tca.xlf index 60ba805e0..88c8811ec 100644 --- a/Resources/Private/Language/locallang_tca.xlf +++ b/Resources/Private/Language/locallang_tca.xlf @@ -17,6 +17,9 @@ Access Control + + Fallback Chain + @@ -149,6 +152,13 @@ Daily cost limit in dollars (0.00 = unlimited) + + Fallback Chain + + + Ordered list of configuration identifiers to retry against when the primary fails with a connection error, 5xx, or 429. JSON format: {"configurationIdentifiers":["claude-sonnet","ollama-local"]}. Leave empty to disable fallback. Does not apply to streaming requests. + + Active diff --git a/Tests/Functional/Repository/LlmConfigurationFallbackChainTest.php b/Tests/Functional/Repository/LlmConfigurationFallbackChainTest.php new file mode 100644 index 000000000..67611267e --- /dev/null +++ b/Tests/Functional/Repository/LlmConfigurationFallbackChainTest.php @@ -0,0 +1,140 @@ + JSON column -> Extbase hydration + * -> DTO. Confirms the Extbase property mapping (snake_case column -> + * camelCase property) works as expected for the new field. + */ +#[CoversClass(LlmConfiguration::class)] +class LlmConfigurationFallbackChainTest extends AbstractFunctionalTestCase +{ + private LlmConfigurationRepository $subject; + private PersistenceManager $persistenceManager; + + protected function setUp(): void + { + parent::setUp(); + + /** @var LlmConfigurationRepository $subject */ + $subject = $this->get(LlmConfigurationRepository::class); + $this->subject = $subject; + + /** @var PersistenceManager $persistenceManager */ + $persistenceManager = $this->get(PersistenceManager::class); + $this->persistenceManager = $persistenceManager; + } + + #[Test] + public function emptyChainPersistsAndHydratesAsEmpty(): void + { + $identifier = 'fb-empty-' . uniqid(); + + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setName('Fallback Empty Test'); + // Not setting chain at all + + $this->subject->add($config); + $this->persistenceManager->persistAll(); + $this->persistenceManager->clearState(); + + $loaded = $this->subject->findOneByIdentifier($identifier); + self::assertInstanceOf(LlmConfiguration::class, $loaded); + self::assertFalse($loaded->hasFallbackChain()); + self::assertTrue($loaded->getFallbackChainDTO()->isEmpty()); + self::assertSame([], $loaded->getFallbackChainDTO()->configurationIdentifiers); + } + + #[Test] + public function populatedChainRoundTripsThroughDatabase(): void + { + $identifier = 'fb-populated-' . uniqid(); + $chain = new FallbackChain(['secondary', 'tertiary']); + + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setName('Fallback Populated Test'); + $config->setFallbackChainDTO($chain); + + $this->subject->add($config); + $this->persistenceManager->persistAll(); + $this->persistenceManager->clearState(); + + $loaded = $this->subject->findOneByIdentifier($identifier); + self::assertInstanceOf(LlmConfiguration::class, $loaded); + self::assertTrue($loaded->hasFallbackChain()); + self::assertSame( + ['secondary', 'tertiary'], + $loaded->getFallbackChainDTO()->configurationIdentifiers, + ); + } + + #[Test] + public function chainOrderIsPreservedThroughPersistence(): void + { + $identifier = 'fb-order-' . uniqid(); + $chain = new FallbackChain(['third', 'first', 'second']); + + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setName('Fallback Order Test'); + $config->setFallbackChainDTO($chain); + + $this->subject->add($config); + $this->persistenceManager->persistAll(); + $this->persistenceManager->clearState(); + + $loaded = $this->subject->findOneByIdentifier($identifier); + self::assertInstanceOf(LlmConfiguration::class, $loaded); + self::assertSame( + ['third', 'first', 'second'], + $loaded->getFallbackChainDTO()->configurationIdentifiers, + ); + } + + #[Test] + public function replacingChainWithEmptyClearsStoredJson(): void + { + $identifier = 'fb-clear-' . uniqid(); + + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setName('Fallback Clear Test'); + $config->setFallbackChainDTO(new FallbackChain(['a', 'b'])); + + $this->subject->add($config); + $this->persistenceManager->persistAll(); + + // Load, clear, save + $loaded = $this->subject->findOneByIdentifier($identifier); + self::assertInstanceOf(LlmConfiguration::class, $loaded); + $loaded->setFallbackChainDTO(new FallbackChain()); + $this->subject->update($loaded); + $this->persistenceManager->persistAll(); + $this->persistenceManager->clearState(); + + $reloaded = $this->subject->findOneByIdentifier($identifier); + self::assertInstanceOf(LlmConfiguration::class, $reloaded); + self::assertFalse($reloaded->hasFallbackChain()); + self::assertSame('', $reloaded->getFallbackChain()); + } +} diff --git a/Tests/Integration/Service/FallbackChainIntegrationTest.php b/Tests/Integration/Service/FallbackChainIntegrationTest.php new file mode 100644 index 000000000..69dbf2e05 --- /dev/null +++ b/Tests/Integration/Service/FallbackChainIntegrationTest.php @@ -0,0 +1,307 @@ + FallbackChainExecutor + * -> ProviderAdapterRegistry -> real provider instances with mocked HTTP. + */ +#[CoversClass(LlmServiceManager::class)] +#[CoversClass(FallbackChainExecutor::class)] +class FallbackChainIntegrationTest extends AbstractIntegrationTestCase +{ + private ExtensionConfiguration&Stub $extensionConfigStub; + private ProviderAdapterRegistry&Stub $adapterRegistryStub; + private LlmConfigurationRepository&Stub $configRepositoryStub; + + protected function setUp(): void + { + parent::setUp(); + + $this->extensionConfigStub = self::createStub(ExtensionConfiguration::class); + $this->extensionConfigStub->method('get') + ->willReturn(['providers' => []]); + + $this->adapterRegistryStub = self::createStub(ProviderAdapterRegistry::class); + $this->configRepositoryStub = self::createStub(LlmConfigurationRepository::class); + } + + #[Test] + public function primaryProviderSuccessSkipsFallbackEntirely(): void + { + $primaryHttp = $this->createHttpClientWithResponses([ + $this->createSuccessResponse($this->getOpenAiChatResponse('from primary')), + ]); + $primaryConfig = $this->buildConfiguration('primary', 'openai', 'gpt-4o', new FallbackChain(['fallback'])); + $openAi = $this->buildOpenAiProvider($primaryHttp); + + $this->adapterRegistryStub->method('createAdapterFromModel') + ->willReturn($openAi); + + $manager = $this->buildManager(); + + $response = $manager->chatWithConfiguration( + [['role' => 'user', 'content' => 'Hello']], + $primaryConfig, + ); + + self::assertSame('from primary', $response->content); + } + + #[Test] + public function connectionErrorOnPrimaryRoutesToFallback(): void + { + // Primary OpenAI returns 503 repeatedly -> ProviderConnectionException + $primaryHttp = $this->createHttpClientWithResponses(array_fill( + 0, + 5, + new Response(503, ['Content-Type' => 'application/json'], '{"error":"service unavailable"}'), + )); + $fallbackHttp = $this->createHttpClientWithResponses([ + $this->createSuccessResponse($this->getClaudeChatResponse('from fallback')), + ]); + + $primaryConfig = $this->buildConfiguration('primary', 'openai', 'gpt-4o', new FallbackChain(['fallback'])); + $fallbackConfig = $this->buildConfiguration('fallback', 'anthropic', 'claude-sonnet-4-20250514'); + + $openAi = $this->buildOpenAiProvider($primaryHttp); + $claude = $this->buildClaudeProvider($fallbackHttp); + + $this->configRepositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['fallback', $fallbackConfig], + ]); + + $this->adapterRegistryStub->method('createAdapterFromModel') + ->willReturnCallback(fn(Model $model) => $model->getModelId() === 'gpt-4o' ? $openAi : $claude); + + $manager = $this->buildManager(); + + $response = $manager->chatWithConfiguration( + [['role' => 'user', 'content' => 'Hello']], + $primaryConfig, + ); + + self::assertSame('from fallback', $response->content); + } + + #[Test] + public function rateLimitOn429RoutesToFallback(): void + { + // OpenAI returns 429 -> ProviderResponseException(code=429) -> retryable + $primaryHttp = $this->createHttpClientWithResponses([ + new Response( + 429, + ['Content-Type' => 'application/json'], + '{"error":{"message":"rate limited","type":"rate_limit"}}', + ), + ]); + $fallbackHttp = $this->createHttpClientWithResponses([ + $this->createSuccessResponse($this->getClaudeChatResponse('from fallback')), + ]); + + $primaryConfig = $this->buildConfiguration('primary', 'openai', 'gpt-4o', new FallbackChain(['fallback'])); + $fallbackConfig = $this->buildConfiguration('fallback', 'anthropic', 'claude-sonnet-4-20250514'); + + $openAi = $this->buildOpenAiProvider($primaryHttp); + $claude = $this->buildClaudeProvider($fallbackHttp); + + $this->configRepositoryStub->method('findOneByIdentifier') + ->willReturnMap([['fallback', $fallbackConfig]]); + + $this->adapterRegistryStub->method('createAdapterFromModel') + ->willReturnCallback(fn(Model $model) => $model->getModelId() === 'gpt-4o' ? $openAi : $claude); + + $manager = $this->buildManager(); + + $response = $manager->chatWithConfiguration( + [['role' => 'user', 'content' => 'Hello']], + $primaryConfig, + ); + + self::assertSame('from fallback', $response->content); + } + + #[Test] + public function clientError4xxDoesNotFallBack(): void + { + // OpenAI returns 401 -> ProviderResponseException(code=401) -> NOT retryable + $primaryHttp = $this->createHttpClientWithResponses([ + new Response( + 401, + ['Content-Type' => 'application/json'], + '{"error":{"message":"invalid api key","type":"authentication_error"}}', + ), + ]); + + $primaryConfig = $this->buildConfiguration('primary', 'openai', 'gpt-4o', new FallbackChain(['fallback'])); + $openAi = $this->buildOpenAiProvider($primaryHttp); + + // Stub fallback anyway so we can assert it's never resolved + $fallbackLookups = 0; + $this->configRepositoryStub->method('findOneByIdentifier') + ->willReturnCallback(function () use (&$fallbackLookups) { + $fallbackLookups++; + return null; + }); + + $this->adapterRegistryStub->method('createAdapterFromModel') + ->willReturn($openAi); + + $manager = $this->buildManager(); + + $this->expectException(ProviderResponseException::class); + try { + $manager->chatWithConfiguration( + [['role' => 'user', 'content' => 'Hello']], + $primaryConfig, + ); + } finally { + self::assertSame(0, $fallbackLookups, 'Fallback should not be consulted on 4xx'); + } + } + + #[Test] + public function allProvidersFailingRaisesExhaustedException(): void + { + $primaryHttp = $this->createHttpClientWithResponses(array_fill( + 0, + 5, + new Response(503, ['Content-Type' => 'application/json'], '{"error":"down"}'), + )); + $fallbackHttp = $this->createHttpClientWithResponses(array_fill( + 0, + 5, + new Response(503, ['Content-Type' => 'application/json'], '{"error":"down"}'), + )); + + $primaryConfig = $this->buildConfiguration('primary', 'openai', 'gpt-4o', new FallbackChain(['fallback'])); + $fallbackConfig = $this->buildConfiguration('fallback', 'anthropic', 'claude-sonnet-4-20250514'); + + $openAi = $this->buildOpenAiProvider($primaryHttp); + $claude = $this->buildClaudeProvider($fallbackHttp); + + $this->configRepositoryStub->method('findOneByIdentifier') + ->willReturnMap([['fallback', $fallbackConfig]]); + + $this->adapterRegistryStub->method('createAdapterFromModel') + ->willReturnCallback(fn(Model $model) => $model->getModelId() === 'gpt-4o' ? $openAi : $claude); + + $manager = $this->buildManager(); + + try { + $manager->chatWithConfiguration( + [['role' => 'user', 'content' => 'Hello']], + $primaryConfig, + ); + self::fail('Expected FallbackChainExhaustedException'); + } catch (FallbackChainExhaustedException $e) { + self::assertSame(['primary', 'fallback'], $e->getAttemptedConfigurations()); + } + } + + private function buildManager(): LlmServiceManager + { + $executor = new FallbackChainExecutor( + $this->configRepositoryStub, + new NullLogger(), + ); + return new LlmServiceManager( + $this->extensionConfigStub, + new NullLogger(), + $this->adapterRegistryStub, + $executor, + ); + } + + private function buildConfiguration( + string $identifier, + string $adapterType, + string $modelId, + ?FallbackChain $chain = null, + ): LlmConfiguration { + $provider = new Provider(); + $provider->setIdentifier($adapterType); + $provider->setAdapterType($adapterType); + + $model = new Model(); + $model->setIdentifier($modelId); + $model->setModelId($modelId); + $model->setProvider($provider); + + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setIsActive(true); + $config->setLlmModel($model); + if ($chain !== null) { + $config->setFallbackChainDTO($chain); + } + return $config; + } + + private function buildOpenAiProvider(ClientInterface&Stub $httpClient): OpenAiProvider + { + $provider = new OpenAiProvider( + $this->requestFactory, + $this->streamFactory, + $this->createNullLogger(), + $this->createVaultServiceMock(), + $this->createSecureHttpClientFactoryMock(), + ); + $provider->configure([ + 'apiKeyIdentifier' => 'sk-test-openai', + 'defaultModel' => 'gpt-4o', + 'maxRetries' => 1, + ]); + $provider->setHttpClient($httpClient); + return $provider; + } + + private function buildClaudeProvider(ClientInterface&Stub $httpClient): ClaudeProvider + { + $provider = new ClaudeProvider( + $this->requestFactory, + $this->streamFactory, + $this->createNullLogger(), + $this->createVaultServiceMock(), + $this->createSecureHttpClientFactoryMock(), + ); + $provider->configure([ + 'apiKeyIdentifier' => 'sk-test-claude', + 'defaultModel' => 'claude-sonnet-4-20250514', + 'maxRetries' => 1, + ]); + $provider->setHttpClient($httpClient); + return $provider; + } +} diff --git a/Tests/Unit/Domain/DTO/FallbackChainTest.php b/Tests/Unit/Domain/DTO/FallbackChainTest.php new file mode 100644 index 000000000..a99a666d2 --- /dev/null +++ b/Tests/Unit/Domain/DTO/FallbackChainTest.php @@ -0,0 +1,356 @@ +configurationIdentifiers); + self::assertTrue($chain->isEmpty()); + self::assertSame(0, $chain->count()); + } + + #[Test] + public function constructorAcceptsIdentifiers(): void + { + $chain = new FallbackChain(['claude-sonnet', 'ollama-local']); + + self::assertSame(['claude-sonnet', 'ollama-local'], $chain->configurationIdentifiers); + self::assertFalse($chain->isEmpty()); + self::assertSame(2, $chain->count()); + } + + #[Test] + public function fromArrayCreatesInstanceWithIdentifiers(): void + { + $chain = FallbackChain::fromArray([ + 'configurationIdentifiers' => ['gpt-4o', 'claude-sonnet'], + ]); + + self::assertSame(['gpt-4o', 'claude-sonnet'], $chain->configurationIdentifiers); + } + + #[Test] + public function fromArrayUsesEmptyListForMissingKey(): void + { + $chain = FallbackChain::fromArray([]); + + self::assertSame([], $chain->configurationIdentifiers); + } + + #[Test] + public function fromArrayDropsDuplicatesPreservingFirstOccurrenceOrder(): void + { + $chain = FallbackChain::fromArray([ + 'configurationIdentifiers' => ['a', 'b', 'a', 'c', 'b'], + ]); + + self::assertSame(['a', 'b', 'c'], $chain->configurationIdentifiers); + } + + #[Test] + public function fromArrayDropsEmptyStrings(): void + { + $chain = FallbackChain::fromArray([ + 'configurationIdentifiers' => ['', 'a', '', 'b'], + ]); + + self::assertSame(['a', 'b'], $chain->configurationIdentifiers); + } + + #[Test] + public function fromJsonCreatesInstanceFromValidJson(): void + { + $json = json_encode([ + 'configurationIdentifiers' => ['alpha', 'beta'], + ], JSON_THROW_ON_ERROR); + + $chain = FallbackChain::fromJson($json); + + self::assertSame(['alpha', 'beta'], $chain->configurationIdentifiers); + } + + #[Test] + public function fromJsonReturnsEmptyForEmptyString(): void + { + $chain = FallbackChain::fromJson(''); + + self::assertTrue($chain->isEmpty()); + } + + #[Test] + public function fromJsonReturnsEmptyForInvalidJson(): void + { + $chain = FallbackChain::fromJson('not valid json'); + + self::assertTrue($chain->isEmpty()); + } + + #[Test] + public function fromJsonReturnsEmptyForNonArrayJson(): void + { + $chain = FallbackChain::fromJson('"just a string"'); + + self::assertTrue($chain->isEmpty()); + } + + #[Test] + public function fromJsonReturnsEmptyForJsonNull(): void + { + $chain = FallbackChain::fromJson('null'); + + self::assertTrue($chain->isEmpty()); + } + + #[Test] + public function toArrayReturnsIdentifiers(): void + { + $chain = new FallbackChain(['a', 'b']); + + self::assertSame( + ['configurationIdentifiers' => ['a', 'b']], + $chain->toArray(), + ); + } + + #[Test] + public function toJsonReturnsValidJsonString(): void + { + $chain = new FallbackChain(['x', 'y']); + + $json = $chain->toJson(); + /** @var array $decoded */ + $decoded = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + + self::assertSame(['x', 'y'], $decoded['configurationIdentifiers']); + } + + #[Test] + public function jsonSerializeReturnsToArrayResult(): void + { + $chain = new FallbackChain(['z']); + + self::assertSame($chain->toArray(), $chain->jsonSerialize()); + } + + #[Test] + public function jsonEncodeUsesJsonSerialize(): void + { + $chain = new FallbackChain(['one', 'two']); + + $json = json_encode($chain, JSON_THROW_ON_ERROR); + /** @var array $decoded */ + $decoded = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + + self::assertSame(['one', 'two'], $decoded['configurationIdentifiers']); + } + + #[Test] + public function fromJsonAndToJsonRoundTrip(): void + { + $original = new FallbackChain(['p', 'q', 'r']); + + $restored = FallbackChain::fromJson($original->toJson()); + + self::assertSame($original->toArray(), $restored->toArray()); + } + + #[Test] + public function containsReturnsTrueForPresentIdentifier(): void + { + $chain = new FallbackChain(['foo', 'bar']); + + self::assertTrue($chain->contains('foo')); + self::assertTrue($chain->contains('bar')); + } + + #[Test] + public function containsReturnsFalseForAbsentIdentifier(): void + { + $chain = new FallbackChain(['foo']); + + self::assertFalse($chain->contains('bar')); + self::assertFalse($chain->contains('')); + } + + #[Test] + public function withLinkAppendsIdentifier(): void + { + $chain = new FallbackChain(['first']); + + $updated = $chain->withLink('second'); + + self::assertSame(['first', 'second'], $updated->configurationIdentifiers); + self::assertSame(['first'], $chain->configurationIdentifiers); + } + + #[Test] + public function withLinkReturnsSameInstanceForDuplicate(): void + { + $chain = new FallbackChain(['first']); + + $updated = $chain->withLink('first'); + + self::assertSame($chain, $updated); + } + + #[Test] + public function withLinkReturnsSameInstanceForEmptyString(): void + { + $chain = new FallbackChain(['first']); + + $updated = $chain->withLink(''); + + self::assertSame($chain, $updated); + } + + #[Test] + public function withLinkPreservesOrderOfExistingLinks(): void + { + $chain = (new FallbackChain()) + ->withLink('a') + ->withLink('b') + ->withLink('c'); + + self::assertSame(['a', 'b', 'c'], $chain->configurationIdentifiers); + } + + #[Test] + public function withoutRemovesIdentifier(): void + { + $chain = new FallbackChain(['a', 'b', 'c']); + + $updated = $chain->without('b'); + + self::assertSame(['a', 'c'], $updated->configurationIdentifiers); + self::assertSame(['a', 'b', 'c'], $chain->configurationIdentifiers); + } + + #[Test] + public function withoutReturnsSameInstanceIfIdentifierNotPresent(): void + { + $chain = new FallbackChain(['a']); + + $updated = $chain->without('not-there'); + + self::assertSame($chain, $updated); + } + + #[Test] + public function withoutReturnsSameInstanceForEmptyString(): void + { + $chain = new FallbackChain(['a']); + + $updated = $chain->without(''); + + self::assertSame($chain, $updated); + } + + #[Test] + public function constructorDoesNotSanitizeDuplicatesDirectly(): void + { + // The constructor trusts its input (readonly property). + // Sanitization happens on fromArray/fromJson entry points only, + // matching other DTOs in the codebase. This test documents that. + $chain = new FallbackChain(['a', 'a']); + + self::assertSame(['a', 'a'], $chain->configurationIdentifiers); + } + + // ────────────────────────────────────────────── + // Normalisation (trim + lowercase) on sanitize entry points + // ────────────────────────────────────────────── + + #[Test] + public function sanitizeNormalisesCasingAndWhitespace(): void + { + $chain = FallbackChain::fromArray([ + 'configurationIdentifiers' => [' CLAUDE ', 'ollama', 'Claude'], + ]); + + // trimmed + lowercased; duplicate after normalisation dropped + self::assertSame(['claude', 'ollama'], $chain->configurationIdentifiers); + } + + #[Test] + public function sanitizeDropsNonStringEntries(): void + { + // Malformed JSON where the list contains arrays/objects/null/ints + // must not throw "Illegal offset type" during sanitize. + $chain = FallbackChain::fromArray([ + 'configurationIdentifiers' => [ + ['nested' => 'array'], + null, + 42, + true, + 'valid', + (object)['x' => 1], + ], + ]); + + self::assertSame(['valid'], $chain->configurationIdentifiers); + } + + #[Test] + public function fromArrayReturnsEmptyWhenConfigurationIdentifiersIsNotAnArray(): void + { + $chain = FallbackChain::fromArray(['configurationIdentifiers' => 'oops']); + + self::assertTrue($chain->isEmpty()); + } + + #[Test] + public function containsIsCaseInsensitiveAndTrimsInput(): void + { + $chain = new FallbackChain(['claude', 'ollama']); + + self::assertTrue($chain->contains('CLAUDE')); + self::assertTrue($chain->contains(' Claude ')); + self::assertTrue($chain->contains('ollama')); + } + + #[Test] + public function withLinkNormalisesInputBeforeAppending(): void + { + $chain = (new FallbackChain(['claude']))->withLink(' OLLAMA '); + + self::assertSame(['claude', 'ollama'], $chain->configurationIdentifiers); + } + + #[Test] + public function withLinkRejectsWhitespaceOnlyInput(): void + { + $chain = new FallbackChain(['claude']); + + $updated = $chain->withLink(" \t\n "); + + self::assertSame($chain, $updated); + } + + #[Test] + public function withoutNormalisesInputBeforeRemoving(): void + { + $chain = new FallbackChain(['claude', 'ollama']); + + $updated = $chain->without(' OLLAMA '); + + self::assertSame(['claude'], $updated->configurationIdentifiers); + } +} diff --git a/Tests/Unit/Provider/Exception/FallbackChainExhaustedExceptionTest.php b/Tests/Unit/Provider/Exception/FallbackChainExhaustedExceptionTest.php new file mode 100644 index 000000000..f7b23574d --- /dev/null +++ b/Tests/Unit/Provider/Exception/FallbackChainExhaustedExceptionTest.php @@ -0,0 +1,115 @@ + 'primary', 'error' => new ProviderConnectionException('boom', 1)], + ['configuration' => 'fallback-a', 'error' => new ProviderConnectionException('boom', 2)], + ['configuration' => 'fallback-b', 'error' => new ProviderConnectionException('boom', 3)], + ]; + + $exception = FallbackChainExhaustedException::fromAttempts($attempts); + + self::assertStringContainsString('primary -> fallback-a -> fallback-b', $exception->getMessage()); + self::assertStringContainsString('3 configuration(s) in the fallback chain failed', $exception->getMessage()); + } + + #[Test] + public function fromAttemptsSetsLastAttemptAsPrevious(): void + { + $last = new ProviderConnectionException('last-error', 99); + $attempts = [ + ['configuration' => 'a', 'error' => new ProviderConnectionException('first', 1)], + ['configuration' => 'b', 'error' => $last], + ]; + + $exception = FallbackChainExhaustedException::fromAttempts($attempts); + + self::assertSame($last, $exception->getPrevious()); + } + + #[Test] + public function fromAttemptsHandlesEmptyAttempts(): void + { + $exception = FallbackChainExhaustedException::fromAttempts([]); + + self::assertStringContainsString('0 configuration(s) in the fallback chain failed', $exception->getMessage()); + self::assertNull($exception->getPrevious()); + self::assertSame([], $exception->getAttemptErrors()); + self::assertSame([], $exception->getAttemptedConfigurations()); + } + + #[Test] + public function getAttemptErrorsReturnsAllAttempts(): void + { + $err1 = new ProviderConnectionException('one', 1); + $err2 = new ProviderConnectionException('two', 2); + $attempts = [ + ['configuration' => 'x', 'error' => $err1], + ['configuration' => 'y', 'error' => $err2], + ]; + + $exception = FallbackChainExhaustedException::fromAttempts($attempts); + + self::assertSame($attempts, $exception->getAttemptErrors()); + } + + #[Test] + public function getAttemptedConfigurationsReturnsIdentifiersInOrder(): void + { + $attempts = [ + ['configuration' => 'alpha', 'error' => new ProviderConnectionException('a', 1)], + ['configuration' => 'beta', 'error' => new ProviderConnectionException('b', 2)], + ['configuration' => 'gamma', 'error' => new ProviderConnectionException('c', 3)], + ]; + + $exception = FallbackChainExhaustedException::fromAttempts($attempts); + + self::assertSame(['alpha', 'beta', 'gamma'], $exception->getAttemptedConfigurations()); + } + + #[Test] + public function constructorAcceptsPreviousExplicitly(): void + { + $previous = new ProviderConnectionException('root cause', 42); + + $exception = new FallbackChainExhaustedException( + 'custom', + 99, + [['configuration' => 'c', 'error' => $previous]], + $previous, + ); + + self::assertSame('custom', $exception->getMessage()); + self::assertSame(99, $exception->getCode()); + self::assertSame($previous, $exception->getPrevious()); + } +} diff --git a/Tests/Unit/Service/FallbackChainExecutorTest.php b/Tests/Unit/Service/FallbackChainExecutorTest.php new file mode 100644 index 000000000..a832342f2 --- /dev/null +++ b/Tests/Unit/Service/FallbackChainExecutorTest.php @@ -0,0 +1,412 @@ +repositoryStub = self::createStub(LlmConfigurationRepository::class); + } + + #[Test] + public function returnsResultFromPrimaryOnSuccess(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['fallback'])); + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + return 'ok'; + }); + + self::assertSame('ok', $result); + self::assertSame(['primary'], $calls); + } + + #[Test] + public function skipsFallbackWhenChainIsEmpty(): void + { + $primary = $this->makeConfig('primary', new FallbackChain()); + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $err = new ProviderConnectionException('down', 0); + + $caught = $this->captureException( + ProviderConnectionException::class, + fn() => $executor->execute($primary, static function () use ($err): never { + throw $err; + }), + ); + + self::assertSame($err, $caught); + } + + #[Test] + public function fallsBackToNextConfigurationOnConnectionError(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['alt'])); + $alt = $this->makeConfig('alt'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturn($alt); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + if ($config->getIdentifier() === 'primary') { + throw new ProviderConnectionException('down', 0); + } + return 'ok-from-alt'; + }); + + self::assertSame('ok-from-alt', $result); + self::assertSame(['primary', 'alt'], $calls); + } + + #[Test] + public function retriesOnRateLimitResponse(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['alt'])); + $alt = $this->makeConfig('alt'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturn($alt); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + + $result = $executor->execute($primary, static function (LlmConfiguration $config) { + if ($config->getIdentifier() === 'primary') { + throw new ProviderResponseException('rate limited', 429); + } + return 'ok'; + }); + + self::assertSame('ok', $result); + } + + #[Test] + public function doesNotRetryOn4xxResponseOtherThan429(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['alt'])); + $alt = $this->makeConfig('alt'); + $this->repositoryStub->method('findOneByIdentifier') + ->willReturn($alt); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $err = new ProviderResponseException('unauthorized', 401); + $caught = $this->captureException( + ProviderResponseException::class, + function () use ($executor, $primary, $err, &$calls): never { + $executor->execute($primary, static function (LlmConfiguration $config) use (&$calls, $err): never { + $calls[] = $config->getIdentifier(); + throw $err; + }); + }, + ); + + self::assertSame($err, $caught); + self::assertSame(['primary'], $calls); + } + + #[Test] + public function doesNotRetryOnUnsupportedFeature(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['alt'])); + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $err = new UnsupportedFeatureException('nope', 1); + $caught = $this->captureException( + UnsupportedFeatureException::class, + function () use ($executor, $primary, $err, &$calls): never { + $executor->execute($primary, static function (LlmConfiguration $config) use (&$calls, $err): never { + $calls[] = $config->getIdentifier(); + throw $err; + }); + }, + ); + + self::assertSame($err, $caught); + self::assertSame(['primary'], $calls); + } + + #[Test] + public function doesNotRetryOnConfigurationException(): void + { + $primary = $this->makeConfig('primary', new FallbackChain(['alt'])); + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $err = new ProviderConfigurationException('missing key', 1); + $caught = $this->captureException( + ProviderConfigurationException::class, + function () use ($executor, $primary, $err, &$calls): never { + $executor->execute($primary, static function (LlmConfiguration $config) use (&$calls, $err): never { + $calls[] = $config->getIdentifier(); + throw $err; + }); + }, + ); + + self::assertSame($err, $caught); + self::assertSame(['primary'], $calls); + } + + #[Test] + public function walksEntireChainUntilOneSucceeds(): void + { + $primary = $this->makeConfig('p', new FallbackChain(['a', 'b', 'c'])); + $a = $this->makeConfig('a'); + $b = $this->makeConfig('b'); + $c = $this->makeConfig('c'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['a', $a], + ['b', $b], + ['c', $c], + ]); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + if (in_array($config->getIdentifier(), ['p', 'a', 'b'], true)) { + throw new ProviderConnectionException('down', 0); + } + return 'ok'; + }); + + self::assertSame('ok', $result); + self::assertSame(['p', 'a', 'b', 'c'], $calls); + } + + #[Test] + public function throwsExhaustedWhenEveryAttemptFails(): void + { + $primary = $this->makeConfig('p', new FallbackChain(['a', 'b'])); + $a = $this->makeConfig('a'); + $b = $this->makeConfig('b'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['a', $a], + ['b', $b], + ]); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + + $exhausted = $this->captureException( + FallbackChainExhaustedException::class, + fn() => $executor->execute($primary, static function (): never { + throw new ProviderConnectionException('down', 0); + }), + ); + + self::assertSame(['p', 'a', 'b'], $exhausted->getAttemptedConfigurations()); + self::assertCount(3, $exhausted->getAttemptErrors()); + self::assertInstanceOf(ProviderConnectionException::class, $exhausted->getPrevious()); + } + + #[Test] + public function skipsMissingFallbackConfigurations(): void + { + $primary = $this->makeConfig('p', new FallbackChain(['missing', 'b'])); + $b = $this->makeConfig('b'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['missing', null], + ['b', $b], + ]); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + if ($config->getIdentifier() === 'p') { + throw new ProviderConnectionException('down', 0); + } + return 'ok'; + }); + + self::assertSame('ok', $result); + self::assertSame(['p', 'b'], $calls); + } + + #[Test] + public function skipsInactiveFallbackConfigurations(): void + { + $primary = $this->makeConfig('p', new FallbackChain(['inactive', 'b'])); + $inactive = $this->makeConfig('inactive', active: false); + $b = $this->makeConfig('b'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['inactive', $inactive], + ['b', $b], + ]); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + if ($config->getIdentifier() === 'p') { + throw new ProviderConnectionException('down', 0); + } + return 'ok'; + }); + + self::assertSame('ok', $result); + self::assertSame(['p', 'b'], $calls); + } + + #[Test] + public function rethrowsPrimaryErrorWhenChainContainsOnlyTheprimary(): void + { + // If the only fallback entry is the primary's own identifier, the + // post-filter chain is empty. Rethrow the primary error verbatim — + // do NOT wrap one attempt as "every configuration failed". + $primary = $this->makeConfig('p', new FallbackChain(['p'])); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + $err = new ProviderConnectionException('down', 0); + + $caught = $this->captureException( + ProviderConnectionException::class, + function () use ($executor, $primary, $err, &$calls): never { + $executor->execute($primary, static function (LlmConfiguration $config) use (&$calls, $err): never { + $calls[] = $config->getIdentifier(); + throw $err; + }); + }, + ); + + self::assertSame($err, $caught); + self::assertSame(['p'], $calls, 'Primary should be the only attempt'); + } + + #[Test] + public function ignoresPrimaryIdentifierAppearingInOwnChain(): void + { + // Defensive: if somebody configures 'p' as its own fallback, don't retry it. + $primary = $this->makeConfig('p', new FallbackChain(['p', 'b'])); + $b = $this->makeConfig('b'); + + $this->repositoryStub->method('findOneByIdentifier') + ->willReturnMap([ + ['p', $primary], + ['b', $b], + ]); + + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $result = $executor->execute($primary, function (LlmConfiguration $config) use (&$calls) { + $calls[] = $config->getIdentifier(); + if ($config->getIdentifier() === 'p') { + throw new ProviderConnectionException('down', 0); + } + return 'ok'; + }); + + self::assertSame('ok', $result); + self::assertSame(['p', 'b'], $calls); + } + + #[Test] + public function nonProviderExceptionIsNotRetryable(): void + { + $primary = $this->makeConfig('p', new FallbackChain(['a'])); + $executor = new FallbackChainExecutor($this->repositoryStub, new NullLogger()); + $calls = []; + + $err = new RuntimeException('unexpected'); + $caught = $this->captureException( + RuntimeException::class, + function () use ($executor, $primary, $err, &$calls): never { + $executor->execute($primary, static function (LlmConfiguration $config) use (&$calls, $err): never { + $calls[] = $config->getIdentifier(); + throw $err; + }); + }, + ); + + self::assertSame($err, $caught); + self::assertSame(['p'], $calls); + } + + private function makeConfig(string $identifier, ?FallbackChain $chain = null, bool $active = true): LlmConfiguration + { + $config = new LlmConfiguration(); + $config->setIdentifier($identifier); + $config->setIsActive($active); + if ($chain !== null) { + $config->setFallbackChainDTO($chain); + } + return $config; + } + + /** + * Run $action and assert it throws an exception of the expected class. + * + * Alternative to wrapping tests in try/catch with self::fail() — PHPStan + * level 10 flags the self::fail() line as unreachable when the closure + * always throws. Returning the caught exception lets callers make + * identity assertions. + * + * @template T of Throwable + * + * @param class-string $expected + * @param callable(): mixed $action + * + * @return T + */ + private function captureException(string $expected, callable $action): Throwable + { + try { + $action(); + } catch (Throwable $caught) { + self::assertInstanceOf($expected, $caught); + return $caught; + } + self::fail(sprintf('Expected %s was not thrown', $expected)); + } +} diff --git a/ext_tables.sql b/ext_tables.sql index 3218948e2..9a9213865 100644 --- a/ext_tables.sql +++ b/ext_tables.sql @@ -129,6 +129,9 @@ CREATE TABLE tx_nrllm_configuration ( max_tokens_per_day int(11) DEFAULT '0' NOT NULL, max_cost_per_day decimal(10,2) DEFAULT '0.00' NOT NULL, + -- Fallback chain (JSON list of configuration identifiers to try on retryable failures) + fallback_chain text, + -- Status is_active tinyint(1) DEFAULT '1' NOT NULL, is_default tinyint(1) DEFAULT '0' NOT NULL,