[RELEASE] Version 9.3.0 with publish page context menu entry, file edge cache invalidation, and much more

Releases: https://projekte.in2code.de/issues/41730
Releases: #81
Releases: #80
Releases: https://projekte.in2code.de/issues/42172
Releases: https://projekte.in2code.de/issues/33332
Releases: https://projekte.in2code.de/issues/36617
Releases: https://projekte.in2code.de/issues/42436
Releases: https://projekte.in2code.de/issues/36616
Resolves: https://projekte.in2code.de/issues/42863

Author: vertexvaar

CHANGELOG.md

@@ -1,5 +1,33 @@
 # In2publish Core Change Log
 
+9.3.0:
+
+- [META] Set the branch alias version number to 9.3.x-dev
+- [META] Set the EM conf version number to 9.3.0
+- [FEATURE] Add option to treat different levels of deletion differently
+- [BUGFIX] Do not retry to search for a site which can not be found
+- [BUGFIX] Correctly detect deleted states of records removed from local
+- [FEATURE] Display removed records with a black background and X-Icon
+- [FEATURE] Add option to treat removal as difference to deletion
+- [FEATURE] Clear caches of all related pages when publishing files
+- [FEATURE] Add file edge cache invalidator
+- [FEATURE] Add publish page option to page tree context menu
+- [FEATURE] Add enable config option for feature contextMenuPublishEntry
+- [FEATURE] Add translations, document dumb menu entry
+- [FEATURE] Ask external voters if a record can be published
+- [WIP][FEATURE] Add a context menu action to publish a page
+- [DOCS] Add known issue about broken foreign file preview urls
+- [BUGFIX] Add missing return type hints in Record/Interface
+- [BUGFIX] Hide duplicate sys_file records
+- [BUGFIX] Skip the attempt to generate a preview URL for page ID 0
+- [TYPO] Fix some typos in FolderRecordFactory developer guide comments
+- [REFACTOR] Move TCA record label construction logic to TcaService
+- [BUGFIX] Add additionWhere only if it's not empty
+- [BUGFIX] Remove error level from performance tests and raise the warning limits
+- [BUGFIX] Use DBAL count method to count instead of SQL function name as column
+- [BUGFIX] Support sql escape sequence in FlexForm foreign_table_where
+- [RELEASE] Version 9.2.0 with env vars, internal_type file_reference and config debug
+
 9.2.0:
 
 - [META] Set the branch alias version number to 9.1.x-dev

Classes/Communication/RemoteProcedureCall/EnvelopeDispatcher.php

@@ -135,6 +135,7 @@ protected function folderExists(array $request): array
                     $files[$fileIdentifier] = [];
                     $files[$fileIdentifier]['hash'] = $driver->hash($fileIdentifier, 'sha1');
                     $files[$fileIdentifier]['info'] = $driver->getFileInfoByIdentifier($fileIdentifier);
+                    // getPublicUrl does not work on CLI and non-public storages https://forge.typo3.org/issues/90330
                     $files[$fileIdentifier]['publicUrl'] = $storage->getPublicUrl($fileObject);
                 }
             }

Classes/Config/Definer/In2publishCoreDefiner.php

@@ -192,6 +192,7 @@ public function getLocalDefinition()
                                  ->addInteger('maximumOverallRecursion', 8)
                                  ->addBoolean('resolvePageRelations', false)
                                  ->addBoolean('includeSysFileReference', false)
+                                 ->addBoolean('treatRemovedAndDeletedAsDifference', false)
                                  ->addArray(
                                      'fal',
                                      Builder::start()
@@ -285,6 +286,15 @@ public function getLocalDefinition()
                                      [IterativeTcaProcessorValidator::class]
                                  )
                       )
+                      ->addArray(
+                          'features',
+                          Builder::start()
+                                 ->addArray(
+                                     'contextMenuPublishEntry',
+                                     Builder::start()
+                                            ->addBoolean('enable', false)
+                                 )
+                      )
                       ->end();
     }
 

Classes/Controller/FileController.php

@@ -36,6 +36,7 @@
 use In2code\In2publishCore\Domain\Model\RecordInterface;
 use In2code\In2publishCore\Domain\Repository\CommonRepository;
 use In2code\In2publishCore\Domain\Service\Publishing\FolderPublisherService;
+use In2code\In2publishCore\Utility\DatabaseUtility;
 use RuntimeException;
 use Throwable;
 use TYPO3\CMS\Core\Messaging\AbstractMessage;
@@ -47,6 +48,9 @@
 use TYPO3\CMS\Extbase\SignalSlot\Exception\InvalidSlotReturnException;
 use TYPO3\CMS\Extbase\Utility\LocalizationUtility;
 
+use function array_key_exists;
+use function count;
+use function in_array;
 use function strpos;
 
 /**

Classes/Domain/Factory/FolderRecordFactory.php

@@ -130,7 +130,7 @@ protected function initializeDependenciesAndGetFolder($identifier): Folder
             // The root level folder is the "real" default and also respects mount points of the current user.
             $localFolder = $localStorage->getRootLevelFolder();
         } else {
-            // This is the normal case. The identifier identifies the folder inclusive its storage.
+            // This is the normal case. The identifier identifies the folder including its storage.
             try {
                 $localFolder = $resourceFactory->getFolderObjectFromCombinedIdentifier($identifier);
             } catch (FolderDoesNotExistException $exception) {
@@ -194,6 +194,9 @@ public function makeInstance($identifier): RecordInterface
         $properties = ['folder_hash' => $hashedIdentifier, 'storage' => $storageUid];
         $files = $this->commonRepository->findByProperties($properties, true, 'sys_file');
 
+        // Identify sys_file entries with identical identifiers and add all duplicates as related record.
+        $files = $this->moveSameSysFileRecordsToRelatedRecords($files);
+
         // FEATURE: mergeSysFileByIdentifier and enableSysFileReferenceUpdate
         if (true === $this->configuration['mergeSysFileByIdentifier']) {
             $files = $this->mergeSysFileByIdentifier($files);
@@ -551,7 +554,7 @@ protected function getFolderInfoByIdentifier(DriverInterface $driver, $identifie
     protected function fixIntersectingIdentifiers(array $diskIdentifiers, array $indexedIdentifiers, array $files)
     {
         foreach (['local' => 'foreign', 'foreign' => 'local'] as $diskSide => $indexSide) {
-            // Find intersecting identifiers. These are identifiers only on one disk and teh opposite database.
+            // Find intersecting identifiers. These are identifiers only on one disk and the opposite database.
             $notIndexedIdentifier = array_diff(
                 $diskIdentifiers[$diskSide],
                 $indexedIdentifiers[$diskSide],
@@ -877,4 +880,29 @@ protected function convertIdentifiers(DriverInterface $driver, array $identifier
         }
         return $identifierList;
     }
+
+    /**
+     * TYPO3 may create more than one sys_file record for an actual file (or different versions of it).
+     * Each sys_file record will be shown as an independent record in the publish files module.
+     * To prevent these duplications, we take every sys_file record which has an identical identifier as any previous
+     * record and attach that record as a related record, so they will be published as one.
+     *
+     * @param RecordInterface[] $files
+     * @return RecordInterface[]
+     */
+    protected function moveSameSysFileRecordsToRelatedRecords(array $files): array
+    {
+        /** @var RecordInterface[] $fileRecords */
+        $fileRecords = [];
+        foreach ($files as $idx => $file) {
+            $localIdentifier = $file->getLocalProperty('identifier');
+            if (isset($fileRecords[$localIdentifier])) {
+                $fileRecords[$localIdentifier]->addRelatedRecord($file);
+                unset($files[$idx]);
+            } else {
+                $fileRecords[$localIdentifier] = $file;
+            }
+        }
+        return $files;
+    }
 }

Classes/Domain/Model/Record.php

@@ -33,9 +33,12 @@
 use In2code\In2publishCore\Config\ConfigContainer;
 use In2code\In2publishCore\Domain\Service\TcaProcessingService;
 use In2code\In2publishCore\Service\Configuration\TcaService;
+use In2code\In2publishCore\Service\Permission\PermissionService;
 use LogicException;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
+use TYPO3\CMS\Extbase\SignalSlot\Dispatcher;
+
 use function array_diff;
 use function array_filter;
 use function array_key_exists;
@@ -442,7 +445,7 @@ public function setForeignProperties(array $foreignProperties): RecordInterface
      * @param string $side
      * @param array $properties
      *
-     * @return $this
+     * @return RecordInterface
      */
     public function setPropertiesBySideIdentifier($side, array $properties): RecordInterface
     {
@@ -861,7 +864,7 @@ public function getMergedProperty($propertyName)
      *
      * @return void
      */
-    public function calculateState()
+    public function calculateState(): void
     {
         if ($this->tableName === 'sys_file' && !isset($this->additionalProperties['recordDatabaseState'])) {
             if ($this->hasLocalProperty('identifier') && $this->hasForeignProperty('identifier')) {
@@ -1246,4 +1249,27 @@ public function getRecordLanguage(): int
 
         return $language;
     }
+
+    public function isPublishable(): bool
+    {
+        if (!$this->isChangedRecursive()) {
+            return false;
+        }
+        $permissionService = GeneralUtility::makeInstance(PermissionService::class);
+        if (!$permissionService->isUserAllowedToPublish()) {
+            return false;
+        }
+        $signalSlotDispatcher = GeneralUtility::makeInstance(Dispatcher::class);
+        $votes = $signalSlotDispatcher->dispatch(
+            RecordInterface::class,
+            'isPublishable',
+            [['yes' => 0, 'no' => 0], $this->tableName, $this->getIdentifier()]
+        );
+        return $votes[0]['yes'] >= $votes[0]['no'];
+    }
+
+    public function isRemovedFromLocalDatabase(): bool
+    {
+        return $this->isForeignRecordDeleted() && !$this->isRecordRepresentByProperties($this->localProperties);
+    }
 }

Classes/Domain/Model/RecordInterface.php

@@ -133,17 +133,17 @@ public function setForeignProperties(array $foreignProperties): RecordInterface;
     /**
      * @return RecordInterface
      */
-    public function setDirtyProperties();
+    public function setDirtyProperties(): RecordInterface;
 
     /**
      * @return mixed
      */
     public function getDirtyProperties();
 
     /**
-     * @return RecordInterface
+     * @return void
      */
-    public function calculateState();
+    public function calculateState(): void;
 
     /**
      * @return bool
@@ -158,17 +158,17 @@ public function isLocalRecordDeleted(): bool;
     /**
      * Returns an identifier unique in the records table.
      *
-     * @return @return int|string
+     * @return int|string
      */
     public function getIdentifier();
 
     /**
      * @param string $side
      * @param array $properties
      *
-     * @return $this
+     * @return RecordInterface
      */
-    public function setPropertiesBySideIdentifier($side, array $properties);
+    public function setPropertiesBySideIdentifier($side, array $properties): RecordInterface;
 
     /**
      * @param string $side
@@ -371,4 +371,8 @@ public function getPageIdentifier(): int;
     public function getSuperordinatePageIdentifier(): int;
 
     public function getRecordLanguage(): int;
+
+    public function isPublishable(): bool;
+
+    public function isRemovedFromLocalDatabase(): bool;
 }

Classes/Domain/Repository/BaseRepository.php

@@ -157,8 +157,10 @@ protected function findPropertiesByProperty(
         $query->getRestrictions()->removeAll();
         $query->select('*')
               ->from($tableName)
-              ->where($query->expr()->like($propertyName, $query->createNamedParameter($propertyValue)))
-              ->andWhere($additionalWhere);
+              ->where($query->expr()->like($propertyName, $query->createNamedParameter($propertyValue)));
+        if (!empty($additionalWhere)) {
+            $query->andWhere($additionalWhere);
+        }
 
         if (!empty($groupBy)) {
             $query->groupBy($groupBy);
@@ -225,8 +227,10 @@ public function findPropertiesByProperties(
         $query = $connection->createQueryBuilder();
         $query->getRestrictions()->removeAll();
         $query->select('*')
-              ->from($tableName)
-              ->andWhere($additionalWhere);
+              ->from($tableName);
+        if (!empty($additionalWhere)) {
+            $query->andWhere($additionalWhere);
+        }
 
         foreach ($properties as $propertyName => $propertyValue) {
             $query->andWhere($query->expr()->like($propertyName, $query->createNamedParameter($propertyValue)));

Classes/Domain/Repository/CommonRepository.php

@@ -31,6 +31,7 @@
  */
 
 use Exception;
+use In2code\In2publishCore\Config\ConfigContainer;
 use In2code\In2publishCore\Domain\Factory\RecordFactory;
 use In2code\In2publishCore\Domain\Model\NullRecord;
 use In2code\In2publishCore\Domain\Model\RecordInterface;
@@ -46,6 +47,7 @@
 use TYPO3\CMS\Core\Configuration\FlexForm\Exception\InvalidTcaException;
 use TYPO3\CMS\Core\Configuration\FlexForm\FlexFormTools;
 use TYPO3\CMS\Core\Database\Connection;
+use TYPO3\CMS\Core\Database\Query\QueryHelper;
 use TYPO3\CMS\Core\Resource\ResourceFactory;
 use TYPO3\CMS\Core\Service\FlexFormService;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -140,6 +142,11 @@ class CommonRepository extends BaseRepository
      */
     protected $signalSlotDispatcher;
 
+    /**
+     * @var ConfigContainer
+     */
+    protected $configContainer = null;
+
     /**
      * @var Connection
      */
@@ -185,6 +192,7 @@ public function __construct(
         $this->resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class);
         $this->taskRepository = GeneralUtility::makeInstance(TaskRepository::class);
         $this->signalSlotDispatcher = GeneralUtility::makeInstance(Dispatcher::class);
+        $this->configContainer = GeneralUtility::makeInstance(ConfigContainer::class);
         $this->identifierFieldName = $identifierFieldName ?: $this->identifierFieldName;
         $this->localDatabase = $localDatabase;
         $this->foreignDatabase = $foreignDatabase;
@@ -1512,11 +1520,9 @@ protected function fetchRelatedRecordsBySelect(
                 if (!empty($columnConfiguration['foreign_table_where'])) {
                     /** @var ReplaceMarkersService $replaceMarkers */
                     $replaceMarkers = GeneralUtility::makeInstance(ReplaceMarkersService::class);
-                    $whereClause = $replaceMarkers->replaceMarkers(
-                        $record,
-                        $columnConfiguration['foreign_table_where'],
-                        $propertyName
-                    );
+                    $foreignTblWhere = $columnConfiguration['foreign_table_where'];
+                    $foreignTblWhere = QueryHelper::quoteDatabaseIdentifiers($this->localDatabase, $foreignTblWhere);
+                    $whereClause = $replaceMarkers->replaceMarkers($record, $foreignTblWhere, $propertyName);
                 }
 
                 $uidArray = [];
@@ -1901,6 +1907,9 @@ protected function isIgnoredRecord(array $localProperties, array $foreignPropert
      */
     protected function isRemovedAndDeletedRecord(array $localProps, array $foreignProps): bool
     {
+        if ($this->configContainer->get('factory.treatRemovedAndDeletedAsDifference')) {
+            return false;
+        }
         return (empty($localProps) && isset($foreignProps['deleted']) && 1 === (int)$foreignProps['deleted'])
                || (empty($foreignProps) && isset($localProps['deleted']) && 1 === (int)$localProps['deleted']);
     }

Classes/Domain/Service/TableConfiguration/LabelService.php

@@ -38,6 +38,7 @@
 use function array_merge;
 use function array_unique;
 use function sprintf;
+use function trim;
 
 /**
  * Class LabelService
@@ -70,27 +71,27 @@ public function __construct()
      *
      * @return string
      */
-    public function getLabelField($record, $stagingLevel = 'local'): string
+    public function getLabelField(RecordInterface $record, $stagingLevel = 'local'): string
     {
-        $tableName = $record->getTableName();
+        $table = $record->getTableName();
 
-        if ($tableName === 'sys_file_reference') {
+        if ($table === 'sys_file_reference') {
             return sprintf(
                 '%d [%d,%d]',
                 $record->getPropertyBySideIdentifier($stagingLevel, 'uid'),
                 $record->getPropertyBySideIdentifier($stagingLevel, 'uid_local'),
                 $record->getPropertyBySideIdentifier($stagingLevel, 'uid_foreign')
             );
         }
-
-        $fields = $this->getLabelFieldsFromTableConfiguration($tableName);
-        foreach ($fields as $field) {
-            $recordProperties = ObjectAccess::getProperty($record, $stagingLevel . 'Properties');
-            if (!empty($recordProperties[$field])) {
-                return (string)$recordProperties[$field];
-            }
+        $row = ObjectAccess::getProperty($record, $stagingLevel . 'Properties');
+        if (empty($row)) {
+            return $this->emptyFieldValue;
+        }
+        $label = $this->tcaService->getRecordLabel($row, $table);
+        if (trim($label) === '') {
+            $label = $this->emptyFieldValue;
         }
-        return $this->emptyFieldValue;
+        return $label;
     }
 
     /**