[Export] Added batching export of resources data

config/config.yaml

@@ -1,4 +1,5 @@
 imports:
+    - { resource: "./config/messenger.yaml" }
     - { resource: "./config/sylius_grid.yaml" }
     - { resource: "./config/sylius_resource.yaml" }
     - { resource: "./config/twig_hooks.yaml" }

config/config/messenger.yaml

@@ -0,0 +1,24 @@
+parameters:
+    env(SYLIUS_IMPORT_EXPORT_MESSENGER_TRANSPORT_EXPORT_COMMAND_DSN): 'sync://'
+    env(SYLIUS_IMPORT_EXPORT_MESSENGER_TRANSPORT_EXPORT_EVENT_DSN): 'sync://'
+
+    sylius_import_export_messenger_transport_export_command_dsn: '%env(resolve:SYLIUS_IMPORT_EXPORT_MESSENGER_TRANSPORT_EXPORT_COMMAND_DSN)%'
+    sylius_import_export_messenger_transport_export_event_dsn: '%env(resolve:SYLIUS_IMPORT_EXPORT_MESSENGER_TRANSPORT_EXPORT_EVENT_DSN)%'
+
+framework:
+    messenger:
+        transports:
+            export_command:
+                dsn: '%sylius_import_export_messenger_transport_export_command_dsn%'
+                failure_transport: 'export_event'
+            export_event:
+                dsn: '%sylius_import_export_messenger_transport_export_event_dsn%'
+        routing:
+            'Sylius\GridImportExport\Messenger\Command\ExportCommand': export_command
+            'Sylius\GridImportExport\Messenger\Event\ExportProcessCompleted': export_event
+        buses:
+            sylius_import_export.export.command_bus:
+                middleware:
+                    - doctrine_transaction
+                    - sylius_import_export.messenger.middleware.export_batch_tracking
+            sylius_import_export.export.event_bus: ~

config/config/sylius_grid.yaml

@@ -7,22 +7,20 @@ sylius_grid:
             export: '@SyliusGridImportExport\admin\import_export\grid\bulk_action\export.html.twig'
 
     grids:
-        sylius_grid_import_export_admin_process:
+        sylius_grid_import_export_admin_process_export:
             driver:
                 name: doctrine/orm
                 options:
-                    class: Sylius\GridImportExport\Entity\Process
+                    class: Sylius\GridImportExport\Entity\ExportProcess
             sorting:
                 createdAt: desc
             fields:
                 createdAt:
                     type: datetime
                     label: sylius.ui.created_at
-                type:
-                    type: twig
-                    label: sylius.ui.type
-                    options:
-                        template: '@SyliusGridImportExport\admin\import_export\grid\field\type.html.twig'
+                format:
+                    type: string
+                    label: sylius_grid_import_export.ui.format
                 resource:
                     type: string
                     label: sylius_grid_import_export.ui.resource
@@ -38,7 +36,7 @@ sylius_grid:
                         label: sylius_grid_import_export.ui.download
                         options:
                             link:
-                                route: sylius_grid_import_export_admin_process_download
+                                route: sylius_grid_import_export_admin_process_export_download
                                 parameters:
                                     uuid: resource.uuid
                     show:

config/config/sylius_resource.yaml

@@ -1,7 +1,7 @@
 sylius_resource:
     resources:
-        sylius_grid_import_export.process:
+        sylius_grid_import_export.process_export:
             driver: doctrine/orm
             classes:
-                model: Sylius\GridImportExport\Entity\Process
-                interface: Sylius\GridImportExport\Entity\ProcessInterface
+                model: Sylius\GridImportExport\Entity\ExportProcess
+                interface: Sylius\GridImportExport\Entity\ExportProcessInterface

config/config/twig_hooks.yaml

@@ -1,47 +1,2 @@
-sylius_twig_hooks:
-    hooks:
-        'sylius_grid_import_export.admin.process.show.content':
-            sections:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections.html.twig'
-                priority: 0
-
-        'sylius_grid_import_export.admin.process.show.content.sections':
-            general:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general.html.twig'
-                priority: 0
-
-        'sylius_grid_import_export.admin.process.show.content.sections.general':
-            header:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/header.html.twig'
-                priority: 100
-            table:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/table.html.twig'
-                priority: 0
-
-        'sylius_grid_import_export.admin.process.show.content.sections.general.header':
-            title:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/header/title.html.twig'
-                priority: 0
-
-        'sylius_grid_import_export.admin.process.show.content.sections.general.table':
-            uuid:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/uuid.html.twig'
-                priority: 500
-            type:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/type.html.twig'
-                priority: 400
-            resource:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/resource.html.twig'
-                priority: 300
-            format:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/format.html.twig'
-                priority: 200
-            status:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/status.html.twig'
-                priority: 150
-            output:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/output.html.twig'
-                priority: 100
-            timestamps:
-                template: '@SyliusGridImportExport/admin/import_export/show/content/sections/general/timestamps.html.twig'
-                priority: 0
+imports:
+    - { resource: 'twig_hooks/**.yaml' }

config/config/twig_hooks/process_export.yaml

@@ -0,0 +1,44 @@
+sylius_twig_hooks:
+    hooks:
+        'sylius_grid_import_export.admin.process_export.show.content':
+            sections:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections.html.twig'
+                priority: 0
+
+        'sylius_grid_import_export.admin.process_export.show.content.sections':
+            general:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general.html.twig'
+                priority: 0
+
+        'sylius_grid_import_export.admin.process_export.show.content.sections.general':
+            header:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/header.html.twig'
+                priority: 100
+            table:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/table.html.twig'
+                priority: 0
+
+        'sylius_grid_import_export.admin.process_export.show.content.sections.general.header':
+            title:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/header/title.html.twig'
+                priority: 0
+
+        'sylius_grid_import_export.admin.process_export.show.content.sections.general.table':
+            uuid:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/uuid.html.twig'
+                priority: 400
+            resource:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/resource.html.twig'
+                priority: 300
+            format:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/format.html.twig'
+                priority: 200
+            status:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/status.html.twig'
+                priority: 150
+            output:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/output.html.twig'
+                priority: 100
+            timestamps:
+                template: '@SyliusGridImportExport/admin/import_export/process_export/show/content/sections/general/timestamps.html.twig'
+                priority: 0

config/doctrine/Process.orm.xml → config/doctrine/ExportProcess.orm.xml

@@ -4,18 +4,21 @@
     xmlns="http://doctrine-project.org/schemas/orm/doctrine-mapping"
     xmlns:gedmo="http://gediminasm.org/schemas/orm/doctrine-extensions-mapping"
 >
-    <mapped-superclass name="Sylius\GridImportExport\Entity\Process" table="sylius_grid_import_export_process">
+    <mapped-superclass name="Sylius\GridImportExport\Entity\ExportProcess" table="sylius_grid_import_export_process_export">
         <id name="uuid" column="uuid" type="string">
             <generator strategy="NONE" />
         </id>
 
-        <field name="type" column="type" type="string" />
         <field name="status" column="status" type="string" />
         <field name="resource" column="resource" type="string" />
+        <field name="grid" column="grid" type="string" />
         <field name="output" column="output" type="text" nullable="true" />
         <field name="errorMessage" column="error_message" type="text" nullable="true" />
         <field name="format" column="format" type="string" />
+        <field name="parameters" column="parameters" type="json" />
         <field name="resourceIds" column="resource_ids" type="json" nullable="true" />
+        <field name="temporaryDataStorage" column="temporary_data_storage" type="string" nullable="true" />
+        <field name="batchesCount" column="batches_count" type="integer" />
         <field name="createdAt" column="created_at" type="datetime">
             <gedmo:timestampable on="create" />
         </field>

config/routes/admin.yaml

@@ -6,31 +6,32 @@ sylius_import_export.admin.resource.export:
         _sylius:
             filterable: true
 
-sylius_grid_import_export_admin_process:
+sylius_grid_import_export_admin_process_export:
     resource: |
-        alias: sylius_grid_import_export.process
+        alias: sylius_grid_import_export.process_export
         section: admin
         templates: "@SyliusAdmin\\shared\\crud"
         except: ['show', 'create', 'update']
         redirect: index
-        grid: sylius_grid_import_export_admin_process
+        grid: sylius_grid_import_export_admin_process_export
         permission: true
+        path: /import-export/exports
     type: sylius.resource
 
-sylius_grid_import_export_admin_process_show:
-    path: /import-export/processes/{id}
+sylius_grid_import_export_admin_process_export_show:
+    path: /import-export/exports/{id}
     methods: [ GET ]
     defaults:
-        _controller: sylius_grid_import_export.controller.process::showAction
+        _controller: sylius_grid_import_export.controller.process_export::showAction
         _sylius:
             section: admin
             template: '@SyliusAdmin/shared/crud/show.html.twig'
             permission: true
             vars:
-                hook_prefix: 'sylius_grid_import_export.admin.process'
+                hook_prefix: 'sylius_grid_import_export.admin.process_export'
 
-sylius_grid_import_export_admin_process_download:
-    path: /import-export/processes/{uuid}/download
+sylius_grid_import_export_admin_process_export_download:
+    path: /import-export/exports/{uuid}/download
     methods: [ GET ]
     defaults:
         _controller: sylius_import_export.controller.download_export_action

config/services.xml

@@ -10,14 +10,38 @@
     </imports>
 
     <services>
+        <service id="sylius_import_export.messenger.command_handler.create_export_process" class="Sylius\GridImportExport\Messenger\Handler\CreateExportProcessHandler">
+            <argument type="service" id="sylius_import_export.factory.process" />
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
+            <argument type="service" id="sylius_import_export.export.command_bus" />
+            <argument type="service" id="sylius_import_export.export.batched_data_manager" />
+
+            <tag name="messenger.message_handler" bus="sylius.command_bus" />
+        </service>
+
         <service id="sylius_import_export.messenger.command_handler.export" class="Sylius\GridImportExport\Messenger\Handler\ExportCommandHandler">
             <argument type="service" id="sylius.resource_registry" />
-            <argument type="service" id="sylius_import_export.custom_factory.process" />
-            <argument type="service" id="sylius_grid_import_export.repository.process" />
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
             <argument type="service" id="sylius_import_export.registry.resource_data_provider" />
             <argument type="service" id="sylius_grid_import_export.exporter_resolver" />
+            <argument type="service" id="sylius_import_export.export.batched_data_manager" />
 
-            <tag name="messenger.message_handler" bus="sylius.command_bus" />
+            <tag name="messenger.message_handler" bus="sylius_import_export.export.command_bus" />
+        </service>
+
+        <service id="sylius_import_export.messenger.event_handler.export_completed" class="Sylius\GridImportExport\Messenger\Handler\ExportCompletedHandler">
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
+            <argument type="service" id="sylius_grid_import_export.exporter_resolver" />
+            <argument type="service" id="sylius_import_export.export.batched_data_manager" />
+
+            <tag name="messenger.message_handler" bus="sylius_import_export.export.event_bus" />
+        </service>
+
+        <service id="sylius_import_export.messenger.middleware.export_batch_tracking" class="Sylius\GridImportExport\Messenger\Middleware\ExportBatchTrackingMiddleware">
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
+            <argument type="service" id="sylius_import_export.export.event_bus" />
+
+            <tag name="messenger.middleware" />
         </service>
 
         <service id="sylius_import_export.twig.component.export_resource" class="Sylius\GridImportExport\Twig\Component\ExportResourceFormComponent">
@@ -55,13 +79,18 @@
         </service>
 
         <service id="sylius_import_export.controller.download_export_action" class="Sylius\GridImportExport\Controller\DownloadExportAction" public="true">
-            <argument type="service" id="sylius_grid_import_export.repository.process" />
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
 
             <tag name="controller.service_arguments" />
         </service>
 
-        <service id="sylius_import_export.custom_factory.process" class="Sylius\GridImportExport\Factory\ProcessFactory" decorates="sylius_grid_import_export.factory.process">
-            <argument type="service" id=".inner" />
+        <service id="sylius_import_export.factory.process" class="Sylius\GridImportExport\Factory\ProcessFactory">
+            <argument type="service" id="sylius_grid_import_export.factory.process_export" />
+        </service>
+
+        <service id="sylius_import_export.export.batched_data_manager" class="Sylius\GridImportExport\Manager\BatchedExportDataManager">
+            <argument type="service" id="sylius_grid_import_export.repository.process_export" />
+            <argument>%sylius_grid_import_export.export_files_directory%</argument>
         </service>
 
         <service id="sylius_grid_import_export.menu.admin_listener" class="Sylius\GridImportExport\Menu\AdminImportExportMenuListener">

src/Controller/DownloadExportAction.php

@@ -13,7 +13,7 @@
 
 namespace Sylius\GridImportExport\Controller;
 
-use Sylius\GridImportExport\Entity\ProcessInterface;
+use Sylius\GridImportExport\Entity\ExportProcessInterface;
 use Sylius\Resource\Doctrine\Persistence\RepositoryInterface;
 use Symfony\Component\HttpFoundation\BinaryFileResponse;
 use Symfony\Component\HttpFoundation\Response;
@@ -22,18 +22,15 @@
 
 final class DownloadExportAction
 {
-    /**
-     * @param RepositoryInterface<ProcessInterface> $processRepository
-     */
+    /** @param RepositoryInterface<ExportProcessInterface> $exportProcessRepository */
     public function __construct(
-        private RepositoryInterface $processRepository,
+        private RepositoryInterface $exportProcessRepository,
     ) {
     }
 
     public function __invoke(string $uuid): Response
     {
-        $process = $this->processRepository->find($uuid);
-
+        $process = $this->exportProcessRepository->find($uuid);
         if (null === $process) {
             throw new NotFoundHttpException(sprintf('Export process "%s" not found', $uuid));
         }

src/Controller/ExportAction.php

@@ -15,7 +15,7 @@
 
 use Sylius\Bundle\ResourceBundle\Controller\ParametersParserInterface;
 use Sylius\Component\Grid\Provider\GridProviderInterface;
-use Sylius\GridImportExport\Messenger\Command\ExportCommand;
+use Sylius\GridImportExport\Messenger\Command\CreateExportProcess;
 use Sylius\GridImportExport\Provider\ResourceIds\ResourcesIdsProviderInterface;
 use Sylius\Resource\Metadata\RegistryInterface;
 use Symfony\Component\Form\FormFactoryInterface;
@@ -62,12 +62,12 @@ public function __invoke(Request $request, string $grid): Response
             $request,
         );
 
-        $this->commandBus->dispatch(new ExportCommand(
+        $this->commandBus->dispatch(new CreateExportProcess(
             resource: $metadata->getAlias(),
-            grid: $grid,
             format: $format,
-            resourceIds: $resourceIds,
+            grid: $grid,
             parameters: $parameters,
+            resourceIds: $resourceIds,
         ));
 
         /** @var Session $session */