Merge pull request #8391 from beberlei/GH-1569-SubselectFetchMode

[GH-1569] Optimize eager fetch for collections to batch query

Author: greg0ire

Diff for: docs/en/reference/working-with-objects.rst

@@ -782,6 +782,23 @@ and these associations are mapped as EAGER, they will automatically
 be loaded together with the entity being queried and is thus
 immediately available to your application.
 
+Eager Loading can also be configured at runtime through
+``AbstractQuery::setFetchMode`` in DQL or Native Queries.
+
+Eager loading for many-to-one and one-to-one associations is using either a
+LEFT JOIN or a second query for fetching the related entity eagerly.
+
+Eager loading for many-to-one associations uses a second query to load
+the collections for several entities at the same time.
+
+When many-to-many, one-to-one or one-to-many associations are eagerly loaded,
+then the global batch size configuration is used to avoid IN(?) queries with
+too many arguments. The default batch size is 100 and can be changed with
+``Configuration::setEagerFetchBatchSize()``.
+
+For eagerly loaded Many-To-Many associations one query has to be made for each
+collection.
+
 By Lazy Loading
 ~~~~~~~~~~~~~~~
 

Diff for: lib/Doctrine/ORM/Configuration.php

@@ -1144,4 +1144,14 @@ public function isRejectIdCollisionInIdentityMapEnabled(): bool
     {
         return $this->_attributes['rejectIdCollisionInIdentityMap'] ?? false;
     }
+
+    public function setEagerFetchBatchSize(int $batchSize = 100): void
+    {
+        $this->_attributes['fetchModeSubselectBatchSize'] = $batchSize;
+    }
+
+    public function getEagerFetchBatchSize(): int
+    {
+        return $this->_attributes['fetchModeSubselectBatchSize'] ?? 100;
+    }
 }

Diff for: lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php

@@ -1264,7 +1264,7 @@ protected function getSelectColumnsSQL()
             }
 
             $isAssocToOneInverseSide = $assoc['type'] & ClassMetadata::TO_ONE && ! $assoc['isOwningSide'];
-            $isAssocFromOneEager     = $assoc['type'] !== ClassMetadata::MANY_TO_MANY && $assoc['fetch'] === ClassMetadata::FETCH_EAGER;
+            $isAssocFromOneEager     = $assoc['type'] & ClassMetadata::TO_ONE && $assoc['fetch'] === ClassMetadata::FETCH_EAGER;
 
             if (! ($isAssocFromOneEager || $isAssocToOneInverseSide)) {
                 continue;

Diff for: lib/Doctrine/ORM/Query/QueryException.php

@@ -204,6 +204,14 @@ public static function iterateWithFetchJoinNotAllowed($assoc)
         );
     }
 
+    public static function eagerFetchJoinWithNotAllowed(string $sourceEntity, string $fieldName): QueryException
+    {
+        return new self(
+            'Associations with fetch-mode=EAGER may not be using WITH conditions in
+             "' . $sourceEntity . '#' . $fieldName . '".'
+        );
+    }
+
     public static function iterateWithMixedResultNotAllowed(): QueryException
     {
         return new self('Iterating a query with mixed results (using scalars) is not supported.');

Diff for: lib/Doctrine/ORM/Query/SqlWalker.php

@@ -1047,7 +1047,9 @@ public function walkJoinAssociationDeclaration($joinAssociationDeclaration, $joi
             }
         }
 
-        $targetTableJoin = null;
+        if ($relation['fetch'] === ClassMetadata::FETCH_EAGER && $condExpr !== null) {
+            throw QueryException::eagerFetchJoinWithNotAllowed($assoc['sourceEntity'], $assoc['fieldName']);
+        }
 
         // This condition is not checking ClassMetadata::MANY_TO_ONE, because by definition it cannot
         // be the owning side and previously we ensured that $assoc is always the owning side of the associations.

Diff for: lib/Doctrine/ORM/UnitOfWork.php

@@ -51,6 +51,7 @@
 use Throwable;
 use UnexpectedValueException;
 
+use function array_chunk;
 use function array_combine;
 use function array_diff_key;
 use function array_filter;
@@ -314,6 +315,9 @@ class UnitOfWork implements PropertyChangedListener
      */
     private $eagerLoadingEntities = [];
 
+    /** @var array<string, array<string, mixed>> */
+    private $eagerLoadingCollections = [];
+
     /** @var bool */
     protected $hasCache = false;
 
@@ -2749,6 +2753,7 @@ public function clear($entityName = null)
             $this->pendingCollectionElementRemovals =
             $this->visitedCollections               =
             $this->eagerLoadingEntities             =
+            $this->eagerLoadingCollections          =
             $this->orphanRemovals                   = [];
         } else {
             Deprecation::triggerIfCalledFromOutside(
@@ -2938,6 +2943,10 @@ public function createEntity($className, array $data, &$hints = [])
                 continue;
             }
 
+            if (! isset($hints['fetchMode'][$class->name][$field])) {
+                $hints['fetchMode'][$class->name][$field] = $assoc['fetch'];
+            }
+
             $targetClass = $this->em->getClassMetadata($assoc['targetEntity']);
 
             switch (true) {
@@ -3001,10 +3010,6 @@ public function createEntity($className, array $data, &$hints = [])
                         break;
                     }
 
-                    if (! isset($hints['fetchMode'][$class->name][$field])) {
-                        $hints['fetchMode'][$class->name][$field] = $assoc['fetch'];
-                    }
-
                     // Foreign key is set
                     // Check identity map first
                     // FIXME: Can break easily with composite keys if join column values are in
@@ -3098,9 +3103,13 @@ public function createEntity($className, array $data, &$hints = [])
                     $reflField = $class->reflFields[$field];
                     $reflField->setValue($entity, $pColl);
 
-                    if ($assoc['fetch'] === ClassMetadata::FETCH_EAGER) {
-                        $this->loadCollection($pColl);
-                        $pColl->takeSnapshot();
+                    if ($hints['fetchMode'][$class->name][$field] === ClassMetadata::FETCH_EAGER) {
+                        if ($assoc['type'] === ClassMetadata::ONE_TO_MANY) {
+                            $this->scheduleCollectionForBatchLoading($pColl, $class);
+                        } elseif ($assoc['type'] === ClassMetadata::MANY_TO_MANY) {
+                            $this->loadCollection($pColl);
+                            $pColl->takeSnapshot();
+                        }
                     }
 
                     $this->originalEntityData[$oid][$field] = $pColl;
@@ -3117,7 +3126,7 @@ public function createEntity($className, array $data, &$hints = [])
     /** @return void */
     public function triggerEagerLoads()
     {
-        if (! $this->eagerLoadingEntities) {
+        if (! $this->eagerLoadingEntities && ! $this->eagerLoadingCollections) {
             return;
         }
 
@@ -3130,11 +3139,69 @@ public function triggerEagerLoads()
                 continue;
             }
 
-            $class = $this->em->getClassMetadata($entityName);
+            $class   = $this->em->getClassMetadata($entityName);
+            $batches = array_chunk($ids, $this->em->getConfiguration()->getEagerFetchBatchSize());
 
-            $this->getEntityPersister($entityName)->loadAll(
-                array_combine($class->identifier, [array_values($ids)])
-            );
+            foreach ($batches as $batchedIds) {
+                $this->getEntityPersister($entityName)->loadAll(
+                    array_combine($class->identifier, [$batchedIds])
+                );
+            }
+        }
+
+        $eagerLoadingCollections       = $this->eagerLoadingCollections; // avoid recursion
+        $this->eagerLoadingCollections = [];
+
+        foreach ($eagerLoadingCollections as $group) {
+            $this->eagerLoadCollections($group['items'], $group['mapping']);
+        }
+    }
+
+    /**
+     * Load all data into the given collections, according to the specified mapping
+     *
+     * @param PersistentCollection[] $collections
+     * @param array<string, mixed>   $mapping
+     * @psalm-param array{targetEntity: class-string, sourceEntity: class-string, mappedBy: string, indexBy: string|null} $mapping
+     */
+    private function eagerLoadCollections(array $collections, array $mapping): void
+    {
+        $targetEntity = $mapping['targetEntity'];
+        $class        = $this->em->getClassMetadata($mapping['sourceEntity']);
+        $mappedBy     = $mapping['mappedBy'];
+
+        $batches = array_chunk($collections, $this->em->getConfiguration()->getEagerFetchBatchSize(), true);
+
+        foreach ($batches as $collectionBatch) {
+            $entities = [];
+
+            foreach ($collectionBatch as $collection) {
+                $entities[] = $collection->getOwner();
+            }
+
+            $found = $this->getEntityPersister($targetEntity)->loadAll([$mappedBy => $entities]);
+
+            $targetClass    = $this->em->getClassMetadata($targetEntity);
+            $targetProperty = $targetClass->getReflectionProperty($mappedBy);
+
+            foreach ($found as $targetValue) {
+                $sourceEntity = $targetProperty->getValue($targetValue);
+
+                $id     = $this->identifierFlattener->flattenIdentifier($class, $class->getIdentifierValues($sourceEntity));
+                $idHash = implode(' ', $id);
+
+                if (isset($mapping['indexBy'])) {
+                    $indexByProperty = $targetClass->getReflectionProperty($mapping['indexBy']);
+                    $collectionBatch[$idHash]->hydrateSet($indexByProperty->getValue($targetValue), $targetValue);
+                } else {
+                    $collectionBatch[$idHash]->add($targetValue);
+                }
+            }
+        }
+
+        foreach ($collections as $association) {
+            $association->setInitialized(true);
+            $association->takeSnapshot();
         }
     }
 
@@ -3165,6 +3232,33 @@ public function loadCollection(PersistentCollection $collection)
         $collection->setInitialized(true);
     }
 
+    /**
+     * Schedule this collection for batch loading at the end of the UnitOfWork
+     */
+    private function scheduleCollectionForBatchLoading(PersistentCollection $collection, ClassMetadata $sourceClass): void
+    {
+        $mapping = $collection->getMapping();
+        $name    = $mapping['sourceEntity'] . '#' . $mapping['fieldName'];
+
+        if (! isset($this->eagerLoadingCollections[$name])) {
+            $this->eagerLoadingCollections[$name] = [
+                'items'   => [],
+                'mapping' => $mapping,
+            ];
+        }
+
+        $owner = $collection->getOwner();
+        assert($owner !== null);
+
+        $id     = $this->identifierFlattener->flattenIdentifier(
+            $sourceClass,
+            $sourceClass->getIdentifierValues($owner)
+        );
+        $idHash = implode(' ', $id);
+
+        $this->eagerLoadingCollections[$name]['items'][$idHash] = $collection;
+    }
+
     /**
      * Gets the identity map of the UnitOfWork.
      *