diff --git a/doc/06_Extending/02_Events.md b/doc/06_Extending/02_Events.md index 6804dec7..0abe97e6 100644 --- a/doc/06_Extending/02_Events.md +++ b/doc/06_Extending/02_Events.md @@ -12,6 +12,8 @@ Listening for events customizes import behaviour without replacing any component | Event | Fired | |---|---| +| `PreInterpretFileEvent` | Before the interpreter starts reading the source file. | +| `PreQueueRowEvent` | For every extracted row, before it is added to the processing queue. | | `DataObject\PreSaveEvent` | Before an imported data object is saved. | | `DataObject\PostSaveEvent` | After an imported data object is saved. | | `DataObject\ProcessElementExceptionEvent` | When processing a record throws an exception. | @@ -23,6 +25,87 @@ that failed, when the failure can be attributed to one. `PostPreparationEvent` exposes the configuration name, the execution type, and whether the source file was interpreted. +## Interpretation-Stage Events + +The two interpretation-stage events customize how the source file turns into import rows without writing a custom +interpreter. Both expose the configuration name and the execution type, and both are also dispatched (with +`isPreview()` returning `true`) when Pimcore Studio renders the source preview and the available mapping columns, so +the configuration UI shows exactly the data an actual import would produce. The preview dispatches `PreQueueRowEvent` +only for the record being displayed - preceding records are not replayed. Listeners that carry state across rows (for +example a group marker taken from an earlier row) should check `isPreview()` and fall back to stateless behavior, as +the carried state is not available in preview mode. + +### PreInterpretFileEvent + +Dispatched before the interpreter validates and reads the source file. `setPath()` replaces the file that gets +interpreted - use it to normalize a file (transcode it, strip a report preamble, rewrite delimiters) while keeping the +standard interpreter. It also marks the start of an interpretation run, which stateful `PreQueueRowEvent` listeners can +use as a reset signal. + +### PreQueueRowEvent + +Dispatched for every row the interpreter extracted, right before the row is added to the processing queue. The listener +receives the row exactly as the interpreter produced it - before the delta check, the resolver's identifier extraction, +and the mapping pipeline - so changed values (including the ID column) affect which element a row resolves to. + +| Method | Purpose | +|---|---| +| `getOriginalRow()` | The row as extracted, unaffected by other listeners. | +| `getRows()` / `setRows(array $rows)` | The rows that will be queued. Set one row to modify it, an empty array to skip it, or multiple rows to fan the source row out into multiple elements. | +| `skipRow(bool $keepInCleanupIdentifierCache = false)` | Skip the row. See the cleanup warning below. | + +```php +namespace App\EventListener; + +use Pimcore\Bundle\DataImporterBundle\Event\PreQueueRowEvent; +use Symfony\Component\EventDispatcher\Attribute\AsEventListener; + +#[AsEventListener] +final class ProductRowListener +{ + public function __invoke(PreQueueRowEvent $event): void + { + if ($event->getConfigName() !== 'my-product-import') { + return; + } + + $row = $event->getOriginalRow(); + + // skip discontinued products, but keep their existing objects + if (($row['status'] ?? '') === 'discontinued') { + $event->skipRow(keepInCleanupIdentifierCache: true); + + return; + } + + // add a computed column that mapping, resolver and location strategies can use + $row['path'] = '/products/' . $row['category'] . '/' . $row['sku']; + + // fan out: one source row per configured sales channel becomes one element each + $rows = []; + foreach (explode(',', $row['channels']) as $channel) { + $rows[] = ['channel' => $channel] + $row; + } + + $event->setRows($rows); + } +} +``` + +:::warning + +When the import uses an active cleanup strategy, every element whose identifier is not seen during interpretation is +deleted or unpublished. A skipped row's element counts as "not seen". Skip rows with +`skipRow(keepInCleanupIdentifierCache: true)` when their existing elements must survive the cleanup. + +::: + +Custom interpreters that extend `AbstractInterpreter` get both events automatically: the interpreter compiler pass +wires the event dispatcher into every tagged interpreter service that provides a `setEventDispatcher()` method, and +the base class dispatches the events in `interpretFile()` and `processImportRow()`. An interpreter that implements +`InterpreterInterface` directly still receives the dispatcher through that setter, but must dispatch +`PreInterpretFileEvent` and `PreQueueRowEvent` itself from its own reading loop. + ## Example Adjust a data object right before it is saved: diff --git a/src/DataSource/Interpreter/AbstractInterpreter.php b/src/DataSource/Interpreter/AbstractInterpreter.php index 13855f6e..fb75e2e2 100644 --- a/src/DataSource/Interpreter/AbstractInterpreter.php +++ b/src/DataSource/Interpreter/AbstractInterpreter.php @@ -15,6 +15,8 @@ use Pimcore\Bundle\ApplicationLoggerBundle\ApplicationLogger; use Pimcore\Bundle\ApplicationLoggerBundle\FileObject; use Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\DeltaChecker\DeltaChecker; +use Pimcore\Bundle\DataImporterBundle\Event\PreInterpretFileEvent; +use Pimcore\Bundle\DataImporterBundle\Event\PreQueueRowEvent; use Pimcore\Bundle\DataImporterBundle\Exception\InvalidInputException; use Pimcore\Bundle\DataImporterBundle\PimcoreDataImporterBundle; use Pimcore\Bundle\DataImporterBundle\Processing\ImportProcessingService; @@ -23,6 +25,7 @@ use Pimcore\Model\Tool\TmpStore; use Pimcore\Tool\Admin; use Psr\Log\LoggerAwareTrait; +use Symfony\Contracts\EventDispatcher\EventDispatcherInterface; /** * @internal @@ -45,6 +48,8 @@ abstract class AbstractInterpreter implements InterpreterInterface protected Resolver $resolver; + protected ?EventDispatcherInterface $eventDispatcher = null; + /** * @var string[] */ @@ -128,11 +133,22 @@ public function setResolver(Resolver $resolver): void $this->resolver = $resolver; } + public function setEventDispatcher(?EventDispatcherInterface $eventDispatcher): void + { + $this->eventDispatcher = $eventDispatcher; + } + public function interpretFile(string $path): bool { $success = false; $this->resetIdentifierCache(); + if ($this->eventDispatcher !== null) { + $event = new PreInterpretFileEvent($this->configName, $this->executionType, $path); + $this->eventDispatcher->dispatch($event); + $path = $event->getPath(); + } + if ($this->fileValid($path)) { $archiveLogMessage = 'Interpreted source file and created queue items.'; $this->doInterpretFileAndCallProcessRow($path); @@ -161,6 +177,37 @@ public function interpretFile(string $path): bool abstract protected function doInterpretFileAndCallProcessRow(string $path): void; protected function processImportRow(array $data) + { + foreach ($this->applyPreQueueRowEvent($data) as $row) { + $this->addRowToQueue($row); + } + } + + /** + * Lets PreQueueRowEvent listeners modify, skip, or fan out the row the interpreter + * extracted. Without a dispatcher or listeners the row passes through unchanged. + * + * @return array the rows to queue + */ + private function applyPreQueueRowEvent(array $data): array + { + if ($this->eventDispatcher === null) { + return [$data]; + } + + $event = new PreQueueRowEvent($this->configName, $this->executionType, $data); + $this->eventDispatcher->dispatch($event); + + if ($event->isRowSkipped() && $event->shouldKeepSkippedRowInIdentifierCache()) { + // Register the skipped row's identifier so an active cleanup strategy does not + // treat its existing element as removed from the source. + $this->addToIdentifierCache($event->getOriginalRow()); + } + + return $event->getRows(); + } + + private function addRowToQueue(array $data): void { $this->assertValidRowEncoding($data); diff --git a/src/DependencyInjection/CompilerPass/InterpreterConfigurationFactoryPass.php b/src/DependencyInjection/CompilerPass/InterpreterConfigurationFactoryPass.php index 74cedf9f..7ed2c695 100644 --- a/src/DependencyInjection/CompilerPass/InterpreterConfigurationFactoryPass.php +++ b/src/DependencyInjection/CompilerPass/InterpreterConfigurationFactoryPass.php @@ -33,6 +33,16 @@ public function process(ContainerBuilder $container): void foreach ($tags as $attributes) { $interpreters[$attributes['type']] = new Reference($id); } + + // Wire the event dispatcher into every interpreter that supports it (built-in + // and custom alike), so PreInterpretFileEvent/PreQueueRowEvent are dispatched. + $definition = $container->getDefinition($id); + $class = $definition->getClass() ?? $id; + if (method_exists($class, 'setEventDispatcher') + && !$definition->hasMethodCall('setEventDispatcher') + ) { + $definition->addMethodCall('setEventDispatcher', [new Reference('event_dispatcher')]); + } } } diff --git a/src/Event/PreInterpretFileEvent.php b/src/Event/PreInterpretFileEvent.php new file mode 100644 index 00000000..5573b992 --- /dev/null +++ b/src/Event/PreInterpretFileEvent.php @@ -0,0 +1,63 @@ +configName; + } + + public function getExecutionType(): string + { + return $this->executionType; + } + + public function getPath(): string + { + return $this->path; + } + + public function setPath(string $path): self + { + $this->path = $path; + + return $this; + } + + /** + * True when the file is being read for the Studio preview instead of an actual import. + */ + public function isPreview(): bool + { + return $this->preview; + } +} diff --git a/src/Event/PreQueueRowEvent.php b/src/Event/PreQueueRowEvent.php new file mode 100644 index 00000000..7659a727 --- /dev/null +++ b/src/Event/PreQueueRowEvent.php @@ -0,0 +1,126 @@ +setRows([$changedRow]); + * - skip: $event->skipRow(); or $event->setRows([]); + * - fan-out: $event->setRows([$rowA, $rowB, $rowC]); + * + * When an import uses an active cleanup strategy, every element whose identifier is not seen + * during interpretation gets deleted or unpublished. Skipping a row therefore makes the + * cleanup treat the row's existing element as removed from the source. Use + * $event->skipRow(keepInCleanupIdentifierCache: true) to skip the row but still register its + * identifier, so the existing element is left untouched. + * + * The same event is dispatched (with isPreview() returning true) when the Studio preview + * renders the source columns, so columns added by listeners are visible and mappable in the + * configuration UI. Rows skipped in preview mode are still displayed unmodified. + */ +final class PreQueueRowEvent extends Event +{ + /** + * @var array + */ + private array $rows; + + private bool $keepSkippedRowInIdentifierCache = false; + + public function __construct( + private readonly string $configName, + private readonly string $executionType, + private readonly array $originalRow, + private readonly bool $preview = false, + ) { + $this->rows = [$originalRow]; + } + + public function getConfigName(): string + { + return $this->configName; + } + + public function getExecutionType(): string + { + return $this->executionType; + } + + /** + * The row as the interpreter extracted it, unaffected by any listener. + */ + public function getOriginalRow(): array + { + return $this->originalRow; + } + + /** + * The rows that will be queued. Initially exactly the original row. + * + * @return array + */ + public function getRows(): array + { + return $this->rows; + } + + /** + * Replace the rows to queue: one row to modify, an empty array to skip, + * multiple rows to fan the source row out into multiple elements. + * + * @param array $rows + */ + public function setRows(array $rows): self + { + $this->rows = array_values($rows); + + return $this; + } + + /** + * Skip this row entirely. With $keepInCleanupIdentifierCache set to true the original + * row's identifier is still registered, so an active cleanup strategy does not treat the + * row's existing element as removed from the source. + */ + public function skipRow(bool $keepInCleanupIdentifierCache = false): self + { + $this->rows = []; + $this->keepSkippedRowInIdentifierCache = $keepInCleanupIdentifierCache; + + return $this; + } + + public function isRowSkipped(): bool + { + return $this->rows === []; + } + + public function shouldKeepSkippedRowInIdentifierCache(): bool + { + return $this->keepSkippedRowInIdentifierCache; + } + + /** + * True when the row is being read for the Studio preview instead of an actual import. + */ + public function isPreview(): bool + { + return $this->preview; + } +} diff --git a/src/Hydrator/PreviewHydrator.php b/src/Hydrator/PreviewHydrator.php index fe1b725b..2372f000 100644 --- a/src/Hydrator/PreviewHydrator.php +++ b/src/Hydrator/PreviewHydrator.php @@ -15,6 +15,7 @@ use Exception; use Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\InterpreterFactory; +use Pimcore\Bundle\DataImporterBundle\Preview\PreviewEventApplier; use Pimcore\Bundle\DataImporterBundle\Preview\PreviewService; use Pimcore\Bundle\DataImporterBundle\Schema\ColumnHeadersResponse; use Pimcore\Bundle\DataImporterBundle\Schema\DataPreviewResponse; @@ -33,7 +34,8 @@ public function __construct( private SecurityServiceInterface $securityService, private PreviewService $previewService, - private InterpreterFactory $interpreterFactory + private InterpreterFactory $interpreterFactory, + private PreviewEventApplier $previewEventApplier ) { } @@ -66,7 +68,17 @@ public function loadAvailableColumnHeaders(string $name, array $config): array $config['interpreterConfig'], $config['processingConfig'] ); + $previewFilePath = $this->previewEventApplier->applyToPath( + $name, + $config['processingConfig'], + $previewFilePath + ); $dataPreview = $interpreter->previewData($previewFilePath); + $dataPreview = $this->previewEventApplier->applyToPreviewData( + $name, + $config['processingConfig'], + $dataPreview + ); $columnHeaders = $dataPreview->getDataColumnHeaders(); if (!$this->isValidJson($columnHeaders)) { diff --git a/src/Preview/PreviewEventApplier.php b/src/Preview/PreviewEventApplier.php new file mode 100644 index 00000000..ac6ff128 --- /dev/null +++ b/src/Preview/PreviewEventApplier.php @@ -0,0 +1,143 @@ +resolveExecutionType($processingConfig), + $path, + preview: true + ); + $this->eventDispatcher->dispatch($event); + + return $event->getPath(); + } + + /** + * Lets PreQueueRowEvent listeners rewrite the preview record, mirroring what happens to + * every row of an actual import. A skipped row stays visible unmodified (the preview is + * an inspection aid). A fan-out displays the first resulting row exactly as it would be + * queued and exposes the column set of all resulting rows as headers, so every column a + * listener adds is mappable; columns no resulting row contains are removed. + * + * Note: the event is dispatched for the displayed record only - preceding records are not + * replayed. Listeners that carry state across rows should use isPreview() to fall back to + * stateless behavior in preview mode. + */ + public function applyToPreviewData( + string $configName, + array $processingConfig, + PreviewData $previewData, + array $mappedColumns = [] + ): PreviewData { + $originalRow = $previewData->getRawData(); + + // an empty preview record produces no row event during a real import either + if ($originalRow === []) { + return $previewData; + } + + $event = new PreQueueRowEvent( + $configName, + $this->resolveExecutionType($processingConfig), + $originalRow, + preview: true + ); + $this->eventDispatcher->dispatch($event); + + $rows = $event->getRows(); + if ($rows === [$originalRow] || $rows === []) { + return $previewData; + } + + // display the first resulting row exactly as it would be queued - no value merging + return new PreviewData( + $this->buildResultLabels($previewData, $rows), + $rows[0], + $previewData->getRecordNumber(), + $mappedColumns + ); + } + + /** + * Restricts the original column headers to the columns the resulting rows actually + * contain (keeping the original order) and adds labels for listener-added columns. + * + * @param array $rows + */ + private function buildResultLabels(PreviewData $previewData, array $rows): array + { + $labels = []; + foreach ($previewData->getDataColumnHeaders() as $columnHeader) { + $labels[$columnHeader['dataIndex']] = $columnHeader['label']; + } + + $resultColumns = []; + foreach ($rows as $resultRow) { + foreach (array_keys($resultRow) as $index) { + $resultColumns[$index] = true; + } + } + + foreach (array_keys($labels) as $index) { + if (!isset($resultColumns[$index]) && !isset($resultColumns[(string) $index])) { + unset($labels[$index]); + } + } + + foreach (array_keys($resultColumns) as $index) { + if (!$this->hasLabel($labels, $index)) { + $labels[$index] = is_int($index) ? "[$index]" : (string) $index; + } + } + + return $labels; + } + + private function hasLabel(array $labels, int|string $index): bool + { + return array_key_exists($index, $labels) || array_key_exists((string) $index, $labels); + } + + private function resolveExecutionType(array $processingConfig): string + { + return $processingConfig['executionType'] ?? ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL; + } +} diff --git a/src/Resources/config/services.yml b/src/Resources/config/services.yml index dadef219..c545d0c9 100644 --- a/src/Resources/config/services.yml +++ b/src/Resources/config/services.yml @@ -23,6 +23,8 @@ services: Pimcore\Bundle\DataImporterBundle\Processing\ExecutionService: ~ Pimcore\Bundle\DataImporterBundle\Settings\ConfigurationPreparationService: ~ Pimcore\Bundle\DataImporterBundle\Preview\PreviewService: ~ + + Pimcore\Bundle\DataImporterBundle\Preview\PreviewEventApplier: ~ Pimcore\Bundle\DataImporterBundle\EventListener\ConfigurationEventSubscriber: ~ Pimcore\Bundle\DataImporterBundle\EventListener\DataImporterListener: tags: diff --git a/src/Service/Studio/PreviewDataService.php b/src/Service/Studio/PreviewDataService.php index df3bdd64..5f9b4354 100644 --- a/src/Service/Studio/PreviewDataService.php +++ b/src/Service/Studio/PreviewDataService.php @@ -19,6 +19,7 @@ use Pimcore\Bundle\DataImporterBundle\Event\Studio\PreResponse\ColumnHeadersEvent; use Pimcore\Bundle\DataImporterBundle\Event\Studio\PreResponse\DataPreviewEvent; use Pimcore\Bundle\DataImporterBundle\Hydrator\PreviewHydratorInterface; +use Pimcore\Bundle\DataImporterBundle\Preview\PreviewEventApplier; use Pimcore\Bundle\DataImporterBundle\Preview\PreviewService; use Pimcore\Bundle\DataImporterBundle\Schema\ColumnHeadersResponse; use Pimcore\Bundle\DataImporterBundle\Schema\DataPreviewResponse; @@ -56,7 +57,8 @@ public function __construct( private ConfigurationPreparationService $configurationPreparationService, private DataLoaderFactory $dataLoaderFactory, private InterpreterFactory $interpreterFactory, - private EventDispatcherInterface $eventDispatcher + private EventDispatcherInterface $eventDispatcher, + private PreviewEventApplier $previewEventApplier ) { } @@ -184,6 +186,12 @@ public function loadPreviewData( $preparedConfig['processingConfig'] ); + $previewFilePath = $this->previewEventApplier->applyToPath( + $name, + $preparedConfig['processingConfig'], + $previewFilePath + ); + if (!$interpreter->fileValid($previewFilePath)) { throw new EnvironmentException( 'Preview file is not valid for the configured interpreter. ' @@ -192,6 +200,12 @@ public function loadPreviewData( } $dataPreview = $interpreter->previewData($previewFilePath, $recordNumber, $mappedColumns); + $dataPreview = $this->previewEventApplier->applyToPreviewData( + $name, + $preparedConfig['processingConfig'], + $dataPreview, + $mappedColumns + ); $preview = $dataPreview->getDataPreview(); if (!$this->previewHydrator->isValidJson($preview)) { diff --git a/src/Service/Studio/TransformationService.php b/src/Service/Studio/TransformationService.php index 4c0ac634..dd7cc078 100644 --- a/src/Service/Studio/TransformationService.php +++ b/src/Service/Studio/TransformationService.php @@ -18,6 +18,7 @@ use Pimcore\Bundle\DataImporterBundle\Event\Studio\PreResponse\TransformationResultTypeEvent; use Pimcore\Bundle\DataImporterBundle\Hydrator\TransformationHydratorInterface; use Pimcore\Bundle\DataImporterBundle\Mapping\MappingConfigurationFactory; +use Pimcore\Bundle\DataImporterBundle\Preview\PreviewEventApplier; use Pimcore\Bundle\DataImporterBundle\Preview\PreviewService; use Pimcore\Bundle\DataImporterBundle\Processing\ImportProcessingService; use Pimcore\Bundle\DataImporterBundle\Schema\TransformationResultPreviewsResponse; @@ -45,7 +46,8 @@ public function __construct( private InterpreterFactory $interpreterFactory, private MappingConfigurationFactory $mappingConfigurationFactory, private ImportProcessingService $importProcessingService, - private EventDispatcherInterface $eventDispatcher + private EventDispatcherInterface $eventDispatcher, + private PreviewEventApplier $previewEventApplier ) { } @@ -76,7 +78,17 @@ public function loadTransformationResultPreviews( $preparedConfig['processingConfig'] ); + $previewFilePath = $this->previewEventApplier->applyToPath( + $name, + $preparedConfig['processingConfig'], + $previewFilePath + ); $dataPreview = $interpreter->previewData($previewFilePath, $recordNumber); + $dataPreview = $this->previewEventApplier->applyToPreviewData( + $name, + $preparedConfig['processingConfig'], + $dataPreview + ); $importDataRow = $dataPreview->getRawData(); } diff --git a/tests/unit/InterpreterCompilerPassTest.php b/tests/unit/InterpreterCompilerPassTest.php new file mode 100644 index 00000000..0574f57e --- /dev/null +++ b/tests/unit/InterpreterCompilerPassTest.php @@ -0,0 +1,80 @@ +setDefinition(InterpreterFactory::class, new Definition(InterpreterFactory::class)); + + $csv = new Definition(CsvFileInterpreter::class); + $csv->addTag(self::INTERPRETER_TAG, ['type' => 'csv']); + $container->setDefinition(CsvFileInterpreter::class, $csv); + + // a tagged interpreter without a setEventDispatcher() method + $plain = new Definition(\stdClass::class); + $plain->addTag(self::INTERPRETER_TAG, ['type' => 'plain']); + $container->setDefinition('test.plain_interpreter', $plain); + + (new InterpreterConfigurationFactoryPass())->process($container); + + return $container; + } + + public function testEventDispatcherIsWiredIntoSupportingInterpreters(): void + { + $container = $this->processContainer(); + + $calls = $container->getDefinition(CsvFileInterpreter::class)->getMethodCalls(); + $dispatcherCalls = array_values(array_filter( + $calls, + static fn (array $call): bool => $call[0] === 'setEventDispatcher' + )); + + $this->assertCount(1, $dispatcherCalls); + $this->assertEquals([new Reference('event_dispatcher')], $dispatcherCalls[0][1]); + } + + public function testInterpretersWithoutTheSetterAreLeftAlone(): void + { + $container = $this->processContainer(); + + $this->assertSame([], $container->getDefinition('test.plain_interpreter')->getMethodCalls()); + } + + public function testAllTaggedInterpretersEndUpInTheFactoryBlueprints(): void + { + $container = $this->processContainer(); + + $bluePrints = $container->getDefinition(InterpreterFactory::class)->getArgument('$interpreterBluePrints'); + + $this->assertEquals( + ['csv' => new Reference(CsvFileInterpreter::class), 'plain' => new Reference('test.plain_interpreter')], + $bluePrints + ); + } +} diff --git a/tests/unit/InterpreterRowEventTest.php b/tests/unit/InterpreterRowEventTest.php new file mode 100644 index 00000000..a774b8ff --- /dev/null +++ b/tests/unit/InterpreterRowEventTest.php @@ -0,0 +1,327 @@ +configName = uniqid('test_row_event_'); + } + + protected function _after(): void + { + $this->drainQueue(); + + foreach ($this->tempFiles as $file) { + @unlink($file); + } + $this->tempFiles = []; + } + + private function queueService(): QueueService + { + return new QueueService(); + } + + private function createInterpreter( + ?EventDispatcherInterface $eventDispatcher, + bool $doCleanup = false + ): CsvFileInterpreter { + $interpreter = new CsvFileInterpreter( + new DeltaChecker(\Pimcore\Db::get()), + $this->queueService(), + ApplicationLogger::getInstance() + ); + $interpreter->setLogger(new NullLogger()); + $interpreter->setConfigName($this->configName); + $interpreter->setExecutionType(ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL); + $interpreter->setIdDataIndex('sku'); + $interpreter->setDoDeltaCheck(false); + $interpreter->setDoCleanup($doCleanup); + if ($doCleanup) { + $interpreter->setResolver($this->createResolver()); + } + $interpreter->setDoArchiveImportFile(false); + $interpreter->setSettings([ + 'skipFirstRow' => true, + 'saveHeaderName' => true, + 'delimiter' => ',', + 'enclosure' => '"', + 'escape' => '\\', + ]); + $interpreter->setEventDispatcher($eventDispatcher); + + return $interpreter; + } + + private function writeCsv(string $content): string + { + $path = tempnam(sys_get_temp_dir(), 'di_csv_') . '.csv'; + file_put_contents($path, $content); + $this->tempFiles[] = $path; + + return $path; + } + + /** + * A resolver whose loading strategy pretends the elements A-1 and B-2 already exist, + * so cleanup behavior can be observed through the cleanup queue items alone. + */ + private function createResolver(): Resolver + { + $resolver = new Resolver(); + $resolver->setLoadingStrategy(new class () implements LoadStrategyInterface { + public function loadElement(array $inputData): ?ElementInterface + { + return null; + } + + public function loadElementByIdentifier($identifier): ?ElementInterface + { + return null; + } + + public function extractIdentifierFromData(array $inputData) + { + return $inputData['sku'] ?? null; + } + + public function loadFullIdentifierList(): array + { + return ['A-1', 'B-2']; + } + + public function setDataObjectClassId($dataObjectClassId): void + { + // the stub resolves identifiers from the row data alone + } + + public function setSettings(array $settings): void + { + // the stub needs no configuration + } + }); + + return $resolver; + } + + /** + * @return array + */ + private function loadQueueEntries(string $jobType): array + { + $queueService = $this->queueService(); + $entries = []; + foreach ($queueService->getAllQueueEntryIds(ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL) as $id) { + $entry = $queueService->getQueueEntryById($id); + if (($entry['configName'] ?? null) === $this->configName && ($entry['jobType'] ?? null) === $jobType) { + $entries[] = $entry; + } + } + + return $entries; + } + + /** + * @return array + */ + private function loadQueuedRows(): array + { + return array_map( + static fn (array $entry): array => json_decode($entry['data'], true), + $this->loadQueueEntries(ImportProcessingService::JOB_TYPE_PROCESS) + ); + } + + private function drainQueue(): void + { + $queueService = $this->queueService(); + foreach ($queueService->getAllQueueEntryIds(ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL) as $id) { + $entry = $queueService->getQueueEntryById($id); + if (($entry['configName'] ?? null) === $this->configName) { + $queueService->markQueueEntryAsProcessed($id); + } + } + } + + public function testSkippedRowWithKeptIdentifierSurvivesCleanup(): void + { + $dispatcher = new EventDispatcher(); + $dispatcher->addListener(PreQueueRowEvent::class, function (PreQueueRowEvent $event): void { + if ($event->getOriginalRow()['sku'] === 'B-2') { + $event->skipRow(keepInCleanupIdentifierCache: true); + } + }); + + $this->createInterpreter($dispatcher, doCleanup: true)->interpretFile($this->writeCsv(self::CSV)); + + $rows = $this->loadQueuedRows(); + $this->assertCount(1, $rows); + $this->assertSame('A-1', $rows[0]['sku']); + $this->assertSame( + [], + $this->loadQueueEntries(ImportProcessingService::JOB_TYPE_CLEANUP), + 'the skipped row kept its identifier, so its existing element must not be cleaned up' + ); + } + + public function testSkippedRowWithoutKeptIdentifierGetsCleanedUp(): void + { + $dispatcher = new EventDispatcher(); + $dispatcher->addListener(PreQueueRowEvent::class, function (PreQueueRowEvent $event): void { + if ($event->getOriginalRow()['sku'] === 'B-2') { + $event->skipRow(); + } + }); + + $this->createInterpreter($dispatcher, doCleanup: true)->interpretFile($this->writeCsv(self::CSV)); + + $cleanupEntries = $this->loadQueueEntries(ImportProcessingService::JOB_TYPE_CLEANUP); + $this->assertCount(1, $cleanupEntries); + $this->assertSame( + 'B-2', + $cleanupEntries[0]['data'], + 'a skipped row without a kept identifier counts as removed from the source' + ); + } + + public function testRowsAreQueuedUnchangedWithoutListeners(): void + { + $interpreter = $this->createInterpreter(new EventDispatcher()); + + $this->assertTrue($interpreter->interpretFile($this->writeCsv(self::CSV))); + + $rows = $this->loadQueuedRows(); + $this->assertCount(2, $rows); + $this->assertSame(['sku' => 'A-1', 'name' => 'First'], $rows[0]); + $this->assertSame(['sku' => 'B-2', 'name' => 'Second'], $rows[1]); + } + + public function testRowsAreQueuedUnchangedWithoutDispatcher(): void + { + $interpreter = $this->createInterpreter(null); + + $this->assertTrue($interpreter->interpretFile($this->writeCsv(self::CSV))); + + $this->assertCount(2, $this->loadQueuedRows()); + } + + public function testListenerCanModifyRows(): void + { + $dispatcher = new EventDispatcher(); + $dispatcher->addListener(PreQueueRowEvent::class, function (PreQueueRowEvent $event): void { + $row = $event->getOriginalRow(); + $row['path'] = '/products/' . $row['sku']; + $event->setRows([$row]); + }); + + $this->createInterpreter($dispatcher)->interpretFile($this->writeCsv(self::CSV)); + + $rows = $this->loadQueuedRows(); + $this->assertCount(2, $rows); + $this->assertSame('/products/A-1', $rows[0]['path']); + $this->assertSame('/products/B-2', $rows[1]['path']); + } + + public function testListenerCanSkipRows(): void + { + $dispatcher = new EventDispatcher(); + $dispatcher->addListener(PreQueueRowEvent::class, function (PreQueueRowEvent $event): void { + if ($event->getOriginalRow()['sku'] === 'B-2') { + $event->skipRow(); + } + }); + + $this->createInterpreter($dispatcher)->interpretFile($this->writeCsv(self::CSV)); + + $rows = $this->loadQueuedRows(); + $this->assertCount(1, $rows); + $this->assertSame('A-1', $rows[0]['sku']); + } + + public function testListenerCanFanOutOneRowIntoMultipleQueueItems(): void + { + $dispatcher = new EventDispatcher(); + $dispatcher->addListener(PreQueueRowEvent::class, function (PreQueueRowEvent $event): void { + $row = $event->getOriginalRow(); + if ($row['sku'] !== 'A-1') { + return; + } + + $rows = []; + foreach ([2023, 2024, 2025] as $year) { + $rows[] = $row + ['year' => $year, 'key' => $row['sku'] . '-' . $year]; + } + $event->setRows($rows); + }); + + $this->createInterpreter($dispatcher)->interpretFile($this->writeCsv(self::CSV)); + + $rows = $this->loadQueuedRows(); + $this->assertCount(4, $rows, 'A-1 fans out into three rows, B-2 stays one row'); + $this->assertSame(['A-1-2023', 'A-1-2024', 'A-1-2025'], array_column(array_slice($rows, 0, 3), 'key')); + $this->assertSame('B-2', $rows[3]['sku']); + } + + public function testPreInterpretFileEventCanReplaceTheSourceFile(): void + { + $originalPath = $this->writeCsv(self::CSV); + $replacementPath = $this->writeCsv("sku,name\nC-3,Replaced\n"); + + $dispatcher = new EventDispatcher(); + $dispatcher->addListener( + PreInterpretFileEvent::class, + function (PreInterpretFileEvent $event) use ($replacementPath): void { + $event->setPath($replacementPath); + } + ); + + $this->createInterpreter($dispatcher)->interpretFile($originalPath); + + $rows = $this->loadQueuedRows(); + $this->assertCount(1, $rows); + $this->assertSame('C-3', $rows[0]['sku']); + } +} diff --git a/tests/unit/PreQueueRowEventTest.php b/tests/unit/PreQueueRowEventTest.php new file mode 100644 index 00000000..a6f32ac0 --- /dev/null +++ b/tests/unit/PreQueueRowEventTest.php @@ -0,0 +1,110 @@ + 'A-1', 'name' => 'First']; + + private function createEvent(): PreQueueRowEvent + { + return new PreQueueRowEvent( + 'test_config', + ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL, + self::ROW + ); + } + + public function testInitialStatePassesRowThroughUnchanged(): void + { + $event = $this->createEvent(); + + $this->assertSame([self::ROW], $event->getRows()); + $this->assertSame(self::ROW, $event->getOriginalRow()); + $this->assertFalse($event->isRowSkipped()); + $this->assertFalse($event->shouldKeepSkippedRowInIdentifierCache()); + $this->assertFalse($event->isPreview()); + } + + public function testSetRowsModifiesTheRow(): void + { + $event = $this->createEvent(); + $modified = self::ROW + ['path' => '/products/A-1']; + + $event->setRows([$modified]); + + $this->assertSame([$modified], $event->getRows()); + $this->assertSame(self::ROW, $event->getOriginalRow(), 'original row must stay untouched'); + $this->assertFalse($event->isRowSkipped()); + } + + public function testSetRowsFansOutAndReindexes(): void + { + $event = $this->createEvent(); + $rowA = self::ROW + ['year' => 2024]; + $rowB = self::ROW + ['year' => 2025]; + + $event->setRows([3 => $rowA, 7 => $rowB]); + + $this->assertSame([$rowA, $rowB], $event->getRows()); + } + + public function testSetRowsWithEmptyArraySkipsTheRow(): void + { + $event = $this->createEvent(); + + $event->setRows([]); + + $this->assertTrue($event->isRowSkipped()); + $this->assertFalse($event->shouldKeepSkippedRowInIdentifierCache()); + } + + public function testSkipRowDefaultsToNotKeepingTheIdentifier(): void + { + $event = $this->createEvent(); + + $event->skipRow(); + + $this->assertSame([], $event->getRows()); + $this->assertTrue($event->isRowSkipped()); + $this->assertFalse($event->shouldKeepSkippedRowInIdentifierCache()); + } + + public function testSkipRowCanKeepTheIdentifierForCleanup(): void + { + $event = $this->createEvent(); + + $event->skipRow(keepInCleanupIdentifierCache: true); + + $this->assertTrue($event->isRowSkipped()); + $this->assertTrue($event->shouldKeepSkippedRowInIdentifierCache()); + } + + public function testPreviewFlag(): void + { + $event = new PreQueueRowEvent( + 'test_config', + ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL, + self::ROW, + preview: true + ); + + $this->assertTrue($event->isPreview()); + } +} diff --git a/tests/unit/PreviewEventApplierTest.php b/tests/unit/PreviewEventApplierTest.php new file mode 100644 index 00000000..e51e22b1 --- /dev/null +++ b/tests/unit/PreviewEventApplierTest.php @@ -0,0 +1,168 @@ + 'A-1', 'name' => 'First']; + + private const LABELS = ['sku' => 'sku', 'name' => 'name']; + + private function createApplier(?callable $rowListener = null, ?callable $fileListener = null): PreviewEventApplier + { + $dispatcher = new EventDispatcher(); + if ($rowListener !== null) { + $dispatcher->addListener(PreQueueRowEvent::class, $rowListener); + } + if ($fileListener !== null) { + $dispatcher->addListener(PreInterpretFileEvent::class, $fileListener); + } + + return new PreviewEventApplier($dispatcher); + } + + private function createPreviewData(): PreviewData + { + return new PreviewData(self::LABELS, self::ROW, 0); + } + + public function testPreviewDataIsReturnedUnchangedWithoutListeners(): void + { + $previewData = $this->createPreviewData(); + + $result = $this->createApplier()->applyToPreviewData('test_config', [], $previewData); + + $this->assertSame($previewData, $result); + } + + public function testListenerAddedColumnsBecomeVisibleAndMappable(): void + { + $applier = $this->createApplier(function (PreQueueRowEvent $event): void { + $this->assertTrue($event->isPreview()); + $row = $event->getOriginalRow(); + $row['path'] = '/products/' . $row['sku']; + $event->setRows([$row]); + }); + + $result = $applier->applyToPreviewData('test_config', [], $this->createPreviewData()); + + $this->assertSame('/products/A-1', $result->getRawData()['path']); + $this->assertContains( + ['id' => 'path', 'dataIndex' => 'path', 'label' => 'path'], + $result->getDataColumnHeaders() + ); + } + + public function testIntegerIndexedColumnsGetTheCsvStyleLabel(): void + { + $applier = $this->createApplier(function (PreQueueRowEvent $event): void { + $event->setRows([[0 => 'A-1', 1 => 'First', 2 => 'synthetic']]); + }); + + $previewData = new PreviewData([0 => 'sku [0]', 1 => 'name [1]'], [0 => 'A-1', 1 => 'First'], 0); + $result = $applier->applyToPreviewData('test_config', [], $previewData); + + $this->assertContains( + ['id' => '2', 'dataIndex' => '2', 'label' => '[2]'], + $result->getDataColumnHeaders() + ); + } + + public function testSkippedRowStaysVisibleInPreview(): void + { + $applier = $this->createApplier(function (PreQueueRowEvent $event): void { + $event->skipRow(); + }); + + $result = $applier->applyToPreviewData('test_config', [], $this->createPreviewData()); + + $this->assertSame(self::ROW, $result->getRawData()); + } + + public function testFanOutShowsFirstRowAndExposesColumnsOfAllRows(): void + { + $applier = $this->createApplier(function (PreQueueRowEvent $event): void { + $row = $event->getOriginalRow(); + $event->setRows([ + $row + ['year' => 2024], + $row + ['year' => 2025, 'extra' => 'x'], + ]); + }); + + $result = $applier->applyToPreviewData('test_config', [], $this->createPreviewData()); + + $this->assertSame(2024, $result->getRawData()['year'], 'first fan-out row is displayed'); + $this->assertArrayNotHasKey( + 'extra', + $result->getRawData(), + 'the displayed row must be exactly the first queued row - no values merged from later rows' + ); + $this->assertContains( + ['id' => 'extra', 'dataIndex' => 'extra', 'label' => 'extra'], + $result->getDataColumnHeaders(), + 'columns unique to later fan-out rows are still mappable' + ); + } + + public function testColumnsRemovedByTheListenerDisappearFromTheHeaders(): void + { + $applier = $this->createApplier(function (PreQueueRowEvent $event): void { + $row = $event->getOriginalRow(); + unset($row['name']); + $event->setRows([$row]); + }); + + $result = $applier->applyToPreviewData('test_config', [], $this->createPreviewData()); + + $this->assertSame(['sku' => 'A-1'], $result->getRawData()); + $this->assertNotContains( + ['id' => 'name', 'dataIndex' => 'name', 'label' => 'name'], + $result->getDataColumnHeaders(), + 'a column no queued row contains must not stay visible or mappable' + ); + } + + public function testEmptyPreviewRecordDispatchesNoEvent(): void + { + $applier = $this->createApplier(function (): void { + $this->fail('a real import emits no row event for an empty source, so preview must not either'); + }); + + $previewData = new PreviewData([], [], -1); + $result = $applier->applyToPreviewData('test_config', [], $previewData); + + $this->assertSame($previewData, $result); + } + + public function testFileListenerCanReplaceThePreviewPath(): void + { + $applier = $this->createApplier(null, function (PreInterpretFileEvent $event): void { + $this->assertTrue($event->isPreview()); + $event->setPath('/tmp/replacement.csv'); + }); + + $this->assertSame( + '/tmp/replacement.csv', + $applier->applyToPath('test_config', [], '/tmp/original.csv') + ); + } +}