[RELEASE] Version 11.0.3 with bug fixes

Releases: #90
Releases: https://projekte.in2code.de/issues/51007
Releases: https://projekte.in2code.de/issues/51055
Releases: https://projekte.in2code.de/issues/51213
Releases: https://projekte.in2code.de/issues/51299
Releases: https://projekte.in2code.de/issues/51301
Releases: https://projekte.in2code.de/issues/51302
Releases: https://projekte.in2code.de/issues/51690
Releases: https://projekte.in2code.de/issues/51693
Releases: https://projekte.in2code.de/issues/51779
Releases: https://projekte.in2code.de/issues/51795
Releases: https://projekte.in2code.de/issues/52187
Resolves: https://projekte.in2code.de/issues/52230

Author: vertexvaar

Build/FunctionalTestsBootstrap.php

@@ -1,7 +1,4 @@
 <?php
-
-use TYPO3\TestingFramework\Core\Testbase;
-
 /*
  * This file is part of the TYPO3 CMS project.
  *

CHANGELOG.md

@@ -1,5 +1,49 @@
 # In2publish Core Change Log
 
+11.0.3:
+
+- [META] Set the EM conf version number to 11.0.3
+- [CLEANUP] Remove useless parentheses
+- [QA] Update QA tools
+- [BUGFIX] Set the correct return type for FileController::getRecordToPublish
+- [COMMENT] Update rFALd::getFoldersInFolder return annotation
+- [BUGFIX] Call the correct (renamed) method rebuildAll for TYPO3 v11
+- [COMMENT] Update the return annotation of findRedirectsByDynamicTarget
+- [BUGFIX] Ensure the rowCount is always an integer
+- [BUGFIX] Don't try to bulkInsert an empty data set
+- [REFACTOR] Reduce cyclomatic and npath complexity
+- [QA] Force installation of unsigned phars (like phpmd) and add all phars to PATH
+- [TESTS] Remove duplicate import of RuntimeException
+- [TESTS] Remove duplicate import of Testbase
+- [BUGFIX] Prevent infinite recursion when collecting records that are going to be published
+- [BUGFIX] Remove the HostNameValidator
+- [BUGFIX] Override database state if sys_file records have been changed
+- [BUGFIX] Early fail the UniqueStorageTargetTest when EXT:scheduler is not installed
+- [BUGFIX] Remove duplicate line which should have been removed
+- [BUGFIX] Prevent undefined array key access in DatabaseUtility
+- [BUGFIX] Set correct flash message severity when file publishing fails and show collected failures
+- [DOCS] Add UPGRADING for v10 to v11 and some publish files module screenshots
+- [TASK] Fix TestLabelLocalizer error
+- [BUGFIX] Add missing slash in combined folder identifier
+- [BUGFIX] Allow input fields with renderType inputLink
+- [BUGFIX] Do not exclude columns which have an itemsProcFunc
+- [BUGFIX] Do not ignore MM-records to excluded tables
+- [BUGFIX] Use the group MM tablenames field instead of true/false
+- [BUGFIX] Fix type error in FileEdgeCacheInvalidationService class
+- [BUGFIX] Fallback to the foreignUid if localUid is null
+- [BUGFIX] Add conditions to not break resetting backend user preferences
+- [BUGFIX] Update the list of required tables
+- [BUGFIX] Fetch all RPC rows instead of just the first
+- [BUGFIX] Allow envelopes to grow arbitrarily huge
+- [TESTS] Add missing ticket annotations for command tests
+- [TESTS] Add functional test to assert that MM records can be marked as publishing
+- [BUGFIX] Allow MM records to be marked as running
+- [BUGFIX] Make RunTasksInQueueCommand compatible with symfony/console 4.4 and add test
+- [TESTS] Fix tests for different dbal and TYPO3 versions
+- [TESTS] Mock Result class which exists in composerMinInstall
+- [TESTS] Fix DbConfigTestCommandTest, remove BackupCommandTest
+- [BUGFIX] Ensure all (testable) commands are executable
+
 11.0.2:
 
 - **Everything included in the 10.2.3 release**
@@ -136,6 +180,36 @@
 - [CLEANUP] Remove outdated compatibility class SignalSlotReplacement
 - [!!!][UPDATE] Require TYPO3 v11
 
+10.2.4:
+
+- [META] Set the EM conf version number to 10.2.4
+- [BUGFIX] Prevent infinite recursion when collecting records that are going to be published
+- [BUGFIX] Remove the HostNameValidator
+- [BUGFIX] Override database state if sys_file records have been changed
+- [BUGFIX] Early fail the UniqueStorageTargetTest when EXT:scheduler is not installed
+- [BUGFIX] Remove duplicate line which should have been removed
+- [BUGFIX] Prevent undefined array key access in DatabaseUtility
+- [TASK] Fix TestLabelLocalizer error
+- [BUGFIX] Allow input fields with renderType inputLink
+- [BUGFIX] Do not exclude columns which have an itemsProcFunc
+- [BUGFIX] Do not ignore MM-records to excluded tables
+- [BUGFIX] Use the group MM tablenames field instead of true/false
+- [BUGFIX] Fix type error in FileEdgeCacheInvalidationService class
+- [BUGFIX] Fallback to the foreignUid if localUid is null
+- [BUGFIX] Add conditions to not break resetting backend user preferences
+- [BUGFIX] Update the list of required tables
+- [BUGFIX] Allow envelopes to grow arbitrarily huge
+- [TESTS] Add missing ticket annotations for command tests
+- [TESTS] Add functional test to assert that MM records can be marked as publishing
+- [BUGFIX] Allow MM records to be marked as running
+- [BUGFIX] Make RunTasksInQueueCommand compatible with symfony/console 4.4 and add test
+- [TESTS] Fix tests for different dbal and TYPO3 versions
+- [TESTS] Mock Result class which exists in composerMinInstall
+- [TESTS] Fix DbConfigTestCommandTest, remove BackupCommandTest
+- [BUGFIX] Ensure all (testable) commands are executable
+- [BUGFIX] Modify FAL folder identifier to work with most 3rd party drivers
+- [RELEASE] Version 10.2.3 with publishing indicator
+
 10.2.3:
 
 - [META] Set the EM conf version number to 10.2.3

Classes/Communication/RemoteProcedureCall/Envelope.php

@@ -32,18 +32,17 @@
 use LogicException;
 
 use function array_filter;
-use function serialize;
-use function unserialize;
 
 class Envelope
 {
     protected int $uid = 0;
 
     protected string $command = '';
 
-    protected string $request = '';
+    protected array $request = [];
 
-    protected string $response = '';
+    /** @var mixed */
+    protected $response = '';
 
     /**
      * @param string $command
@@ -94,26 +93,26 @@ public function setCommand(string $command): void
      */
     public function getRequest()
     {
-        return unserialize($this->request);
+        return $this->request;
     }
 
     /**
      * @param mixed $request
      */
     public function setRequest(array $request): void
     {
-        if ('' !== $this->request) {
+        if ([] !== $this->request) {
             throw new LogicException('Can not overrule an envelope\'s request', 1474386975);
         }
-        $this->request = serialize($request);
+        $this->request = $request;
     }
 
     /**
      * @return mixed
      */
     public function getResponse()
     {
-        return unserialize($this->response);
+        return $this->response;
     }
 
     /**
@@ -124,7 +123,7 @@ public function setResponse($response): void
         if ('' !== $this->response) {
             throw new LogicException('Can not overrule an envelope\'s response', 1474386986);
         }
-        $this->response = serialize($response);
+        $this->response = $response;
     }
 
     public function toArray(): array

Classes/Communication/RemoteProcedureCall/Letterbox.php

@@ -39,14 +39,17 @@
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\SingletonInterface;
 
+use function is_string;
+use function serialize;
+use function str_split;
+use function unserialize;
+
 class Letterbox implements LoggerAwareInterface, SingletonInterface
 {
     use LoggerAwareTrait;
 
-    public const TABLE = 'tx_in2code_in2publish_envelope';
-
+    protected const CHUNK_SIZE = 65000;
     protected ContextService $contextService;
-
     protected bool $keepEnvelopes;
 
     public function __construct(ContextService $contextService, ConfigContainer $configContainer)
@@ -73,14 +76,9 @@ public function sendEnvelope(Envelope $envelope)
 
         $uid = $envelope->getUid();
 
-        if (0 === $uid || 0 === $database->count('uid', static::TABLE, ['uid' => $uid])) {
+        if (0 === $uid || 0 === $database->count('uid', 'tx_in2code_rpc_request', ['uid' => $uid])) {
             try {
-                $database->insert(static::TABLE, $envelope->toArray());
-                if ($uid <= 0) {
-                    $uid = (int)$database->lastInsertId();
-                    $envelope->setUid($uid);
-                }
-                return $uid;
+                return $this->insertNewEnvelope($database, $envelope);
             } catch (Throwable $exception) {
                 $this->logger->error(
                     'Failed to send envelope [' . $uid . ']',
@@ -91,7 +89,7 @@ public function sendEnvelope(Envelope $envelope)
         }
 
         try {
-            $database->update(static::TABLE, $envelope->toArray(), ['uid' => $uid]);
+            $this->updateEnvelope($database, $envelope, $uid);
             return true;
         } catch (Throwable $exception) {
             $this->logger->error(
@@ -124,23 +122,47 @@ public function receiveEnvelope(int $uid, bool $burnEnvelope = true)
 
         $query = $database->createQueryBuilder();
         $query->getRestrictions()->removeAll();
-        $query->select('command', 'request', 'response', 'uid')
-              ->from(static::TABLE)
+        $query->select('req.uid', 'req.command', 'data.payload', 'data.data_type', 'data.sorting')
+              ->from('tx_in2code_rpc_request', 'req')
+              ->join('req', 'tx_in2code_rpc_data', 'data', 'req.uid = data.request')
               ->where($query->expr()->eq('uid', $uid))
-              ->setMaxResults(1);
+              ->orderBy('data.sorting');
         try {
             $result = $query->execute();
-            $envelopeData = $result->fetchAssociative();
+            $rows = $result->fetchAllAssociative();
         } catch (Throwable $exception) {
             $this->logger->error(
                 'Failed to receive envelope [' . $uid . '] "' . $exception . '"',
                 ['exception' => $exception]
             );
             return false;
         }
-        $envelope = Envelope::fromArray($envelopeData);
+
+        $uid = $rows[0]['uid'];
+        $command = $rows[0]['command'];
+
+        $request = '';
+        $response = null;
+        foreach ($rows as $row) {
+            if ('request' === $row['data_type']) {
+                $request .= $row['payload'];
+            }
+            if ('response' === $row['data_type']) {
+                if (null === $response) {
+                    $response = '';
+                }
+                $response .= $row['payload'];
+            }
+        }
+
+        if (is_string($response)) {
+            $response = unserialize($response);
+        }
+        $envelope = new Envelope($command, unserialize($request), $response, $uid);
+
         if (!$this->keepEnvelopes && $burnEnvelope) {
-            $database->delete(static::TABLE, ['uid' => $uid]);
+            $database->delete('tx_in2code_rpc_request', ['uid' => $uid]);
+            $database->delete('tx_in2code_rpc_data', ['request' => $uid]);
         }
         return $envelope;
     }
@@ -156,7 +178,10 @@ public function hasUnAnsweredEnvelopes(): bool
         if ($database instanceof Connection && $database->isConnected()) {
             $query = $database->createQueryBuilder();
             $query->getRestrictions()->removeAll();
-            $query->count('uid')->from(static::TABLE)->where($query->expr()->isNotNull('response'));
+            $query->count('uid')
+                  ->from('tx_in2code_rpc_request', 'req')
+                  ->join('req', 'tx_in2code_rpc_data', 'data', 'req.uid = data.request')
+                  ->where($query->expr()->eq('data.data_type', $query->createNamedParameter('response')));
             return $query->execute()->fetchOne() > 0;
         }
         return false;
@@ -171,6 +196,53 @@ public function removeAnsweredEnvelopes(): void
         }
         $query = $database->createQueryBuilder();
         $query->getRestrictions()->removeAll();
-        $query->delete(static::TABLE)->where($query->expr()->isNotNull('response'))->execute();
+        $query->select('uid')
+              ->from('tx_in2code_rpc_request', 'req')
+              ->join('req', 'tx_in2code_rpc_data', 'data', 'req.uid = data.request')
+              ->where($query->expr()->eq('data.data_type', $query->createNamedParameter('response')));
+        $uid = $query->execute()->fetchColumn();
+        $database->delete('tx_in2code_rpc_request', ['uid' => $uid]);
+        $database->delete('tx_in2code_rpc_data', ['request' => $uid]);
+    }
+
+    protected function insertNewEnvelope(Connection $database, Envelope $envelope): int
+    {
+        $database->beginTransaction();
+
+        $database->insert('tx_in2code_rpc_request', [
+            'command' => $envelope->getCommand(),
+        ]);
+        $uid = (int)$database->lastInsertId();
+
+        $request = serialize($envelope->getRequest());
+        $chunks = str_split($request, self::CHUNK_SIZE);
+        foreach ($chunks as $sorting => $chunk) {
+            $database->insert('tx_in2code_rpc_data', [
+                'request' => $uid,
+                'data_type' => 'request',
+                'payload' => $chunk,
+                'sorting' => $sorting,
+            ]);
+        }
+
+        $database->commit();
+        return $uid;
+    }
+
+    protected function updateEnvelope(Connection $database, Envelope $envelope, int $uid): void
+    {
+        $database->beginTransaction();
+
+        $response = serialize($envelope->getResponse());
+        $chunks = str_split($response, self::CHUNK_SIZE);
+        foreach ($chunks as $sorting => $chunk) {
+            $database->insert('tx_in2code_rpc_data', [
+                'request' => $uid,
+                'data_type' => 'response',
+                'payload' => $chunk,
+                'sorting' => $sorting,
+            ]);
+        }
+        $database->commit();
     }
 }

Classes/Component/FalHandling/Finder/DefaultFalFinder.php

@@ -57,6 +57,7 @@
 use function array_map;
 use function array_merge;
 use function array_values;
+use function ltrim;
 use function sprintf;
 
 /**
@@ -498,7 +499,7 @@ protected function getFolderInfoByIdentifier(DriverInterface $driver, string $id
         if ($driver->folderExists($identifier)) {
             $info = $driver->getFolderInfoByIdentifier($identifier);
             $info['identifier'] = trim($info['identifier'], '/') . '/';
-            $info['uid'] = sprintf('%d:%s', $info['storage'], $info['identifier']);
+            $info['uid'] = sprintf('%d:/%s', $info['storage'], ltrim($info['identifier'], '/'));
         } else {
             $info = [];
         }

Classes/Component/FalHandling/Finder/Factory/FileIndexFactory.php

@@ -126,9 +126,15 @@ public function updateFileIndexInfo(
         string $foreignIdentifier
     ): void {
         $uid = $record->getIdentifier();
-        $record->addAdditionalProperty('recordDatabaseState', $record->getState());
         $localFileInfo = $this->getFileIndexArray($localIdentifier, 'local', $uid);
         $foreignFileInfo = $this->getFileIndexArray($foreignIdentifier, 'foreign', $uid);
+
+        $originalState = $record->getState();
+        if (isset($localFileInfo['uid'], $foreignFileInfo['uid'])) {
+            $originalState = RecordInterface::RECORD_STATE_CHANGED;
+        }
+        $record->addAdditionalProperty('recordDatabaseState', $originalState);
+
         // only set new file info if the file exists at least on one side, else we only deal with the database record
         if (!empty($localFileInfo) || !empty($foreignFileInfo)) {
             $record->setLocalProperties($localFileInfo);