perf(metadata): Add optimized sharding for metadata deletion

Signed-off-by: Carl Schwan <[email protected]>

Author: Carl Schwan

lib/private/Files/Cache/Cache.php

@@ -614,17 +614,17 @@ private function removeChildren(ICacheEntry $entry) {
 
 		$cacheEntryRemovedEvents = [];
 		foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
-			$cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
+			$cacheEntryRemovedEvents[] = new CacheEntryRemovedEvent(
 				$this->storage,
 				$filePath,
 				$fileId,
 				$this->getNumericStorageId()
 			);
-			$cacheEntryRemovedEvents[] = $cacheEntryRemovedEvent;
-			$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
 		}
 		$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents));
-
+		foreach ($cacheEntryRemovedEvents as $cacheEntryRemovedEvent) {
+			$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
+		}
 	}
 
 	/**

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -214,9 +214,9 @@ public function deleteMetadata(int $fileId): void {
 		}
 	}
 
-	public function deleteMetadataForFiles(array $fileIds): void {
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void {
 		try {
-			$this->metadataRequestService->dropMetadataForFiles($fileIds);
+			$this->metadataRequestService->dropMetadataForFiles($storage, $fileIds);
 		} catch (Exception $e) {
 			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
 		}

lib/private/FilesMetadata/Listener/MetadataDelete.php

@@ -33,18 +33,21 @@ public function handle(Event $event): void {
 		}
 
 		$entries = $event->getCacheEntryRemovedEvents();
-		$fileIds = [];
+		$storageToFileIds = [];
 
 		foreach ($entries as $entry) {
 			try {
-				$fileIds[] = $entry->getFileId();
+				$storageToFileIds[$entry->getStorageId()] ??= [];
+				$storageToFileIds[$entry->getStorageId()][] = $entry->getFileId();
 			} catch (Exception $e) {
 				$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
 			}
 		}
 
 		try {
-			$this->filesMetadataManager->deleteMetadataForFiles($fileIds);
+			foreach ($storageToFileIds as $storageId => $fileIds) {
+				$this->filesMetadataManager->deleteMetadataForFiles($storageId, $fileIds);
+			}
 		} catch (Exception $e) {
 			$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
 		}

lib/private/FilesMetadata/Service/MetadataRequestService.php

@@ -147,16 +147,16 @@ public function dropMetadata(int $fileId): void {
 
 	/**
 	 * @param int[] $fileIds
-	 * @return void
 	 * @throws Exception
 	 */
-	public function dropMetadataForFiles(array $fileIds): void {
+	public function dropMetadataForFiles(int $storage, array $fileIds): void {
 		$chunks = array_chunk($fileIds, 1000);
 
 		foreach ($chunks as $chunk) {
 			$qb = $this->dbConnection->getQueryBuilder();
 			$qb->delete(self::TABLE_METADATA)
-				->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
+				->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+				->hintShardKey('storage', $storage);
 			$qb->executeStatement();
 		}
 	}

lib/public/Files/Cache/CacheEntryRemovedEvent.php

@@ -11,7 +11,7 @@
 /**
  * Event for when an existing entry in the cache gets removed
  *
- * Prefer using \c CacheEntriesRemovedEvent as it is more efficient when deleting
+ * Prefer using CacheEntriesRemovedEvent as it is more efficient when deleting
  * multiple files at the same time.
  *
  * @since 21.0.0

lib/public/FilesMetadata/IFilesMetadataManager.php

@@ -103,11 +103,12 @@ public function deleteMetadata(int $fileId): void;
 	/**
 	 * Delete metadata and its indexes of multiple file ids
 	 *
+	 * @param int $storage The storage id coresponding to the $fileIds
 	 * @param array<int> $fileIds file ids
 	 * @return void
 	 * @since 32.0.0
 	 */
-	public function deleteMetadataForFiles(array $fileIds): void;
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void;
 
 	/**
 	 * generate and return a MetadataQuery to help building sql queries