Skip to content

Commit 281a304

Browse files
author
Carl Schwan
committed
feat(filecache): Scale DB query created when deleting file from filecache
Instead of creating a CacheEntryRemovedEvent for each deleted files, create a single CacheEntriesRemovedEvent which wrap multiple CacheEntryRemovedEvent. This allow listener to optimize the query they do when multiple files are deleted at the same time (e.g. when deleting a folder). Signed-off-by: Carl Schwan <[email protected]>
1 parent 1091e59 commit 281a304

File tree

13 files changed

+181
-19
lines changed

13 files changed

+181
-19
lines changed

‎apps/files/lib/AppInfo/Application.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
use OCP\AppFramework\Bootstrap\IRegistrationContext;
3838
use OCP\Collaboration\Reference\RenderReferenceEvent;
3939
use OCP\Collaboration\Resources\IProviderManager;
40-
use OCP\Files\Cache\CacheEntryRemovedEvent;
40+
use OCP\Files\Cache\CacheEntriesRemovedEvent;
4141
use OCP\Files\Events\Node\BeforeNodeCopiedEvent;
4242
use OCP\Files\Events\Node\BeforeNodeDeletedEvent;
4343
use OCP\Files\Events\Node\BeforeNodeRenamedEvent;
@@ -117,7 +117,7 @@ public function register(IRegistrationContext $context): void {
117117
$context->registerEventListener(RenderReferenceEvent::class, RenderReferenceEventListener::class);
118118
$context->registerEventListener(BeforeNodeRenamedEvent::class, SyncLivePhotosListener::class);
119119
$context->registerEventListener(BeforeNodeDeletedEvent::class, SyncLivePhotosListener::class);
120-
$context->registerEventListener(CacheEntryRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted.
120+
$context->registerEventListener(CacheEntriesRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted.
121121
$context->registerEventListener(BeforeNodeCopiedEvent::class, SyncLivePhotosListener::class);
122122
$context->registerEventListener(NodeCopiedEvent::class, SyncLivePhotosListener::class);
123123
$context->registerEventListener(LoadSearchPlugins::class, LoadSearchPluginsListener::class);

‎apps/files/lib/Listener/SyncLivePhotosListener.php‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
use OCP\EventDispatcher\Event;
1818
use OCP\EventDispatcher\IEventListener;
1919
use OCP\Exceptions\AbortedEventException;
20-
use OCP\Files\Cache\CacheEntryRemovedEvent;
20+
use OCP\Files\Cache\CacheEntriesRemovedEvent;
2121
use OCP\Files\Events\Node\BeforeNodeCopiedEvent;
2222
use OCP\Files\Events\Node\BeforeNodeDeletedEvent;
2323
use OCP\Files\Events\Node\BeforeNodeRenamedEvent;
@@ -63,8 +63,8 @@ public function handle(Event $event): void {
6363
$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource()->getId());
6464
} elseif ($event instanceof BeforeNodeDeletedEvent) {
6565
$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getNode()->getId());
66-
} elseif ($event instanceof CacheEntryRemovedEvent) {
67-
$peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getFileId());
66+
} elseif ($event instanceof CacheEntriesRemovedEvent) {
67+
$this->handleCacheEntriesRemovedEvent($event);
6868
}
6969

7070
if ($peerFileId === null) {
@@ -83,12 +83,30 @@ public function handle(Event $event): void {
8383
$this->handleMove($event->getSource(), $event->getTarget(), $peerFile);
8484
} elseif ($event instanceof BeforeNodeDeletedEvent) {
8585
$this->handleDeletion($event, $peerFile);
86-
} elseif ($event instanceof CacheEntryRemovedEvent) {
87-
$peerFile->delete();
8886
}
8987
}
9088
}
9189

90+
public function handleCacheEntriesRemovedEvent(CacheEntriesRemovedEvent $cacheEntriesRemovedEvent): void {
91+
$entries = $cacheEntriesRemovedEvent->getCacheEntryRemovedEvents();
92+
$fileIds = [];
93+
foreach ($entries as $entry) {
94+
$fileIds[] = $entry->getFileId();
95+
}
96+
97+
$peerFileIds = $this->livePhotosService->getLivePhotoPeerIds($fileIds);
98+
99+
foreach ($peerFileIds as $peerFileId) {
100+
// Check the user's folder.
101+
$peerFile = $this->userFolder->getFirstNodeById($peerFileId);
102+
103+
if ($peerFile === null) {
104+
return; // Peer file not found.
105+
}
106+
$peerFile->delete();
107+
}
108+
}
109+
92110
private function runMoveOrCopyChecks(Node $sourceFile, Node $targetFile, Node $peerFile): void {
93111
$targetParent = $targetFile->getParent();
94112
$sourceExtension = $sourceFile->getExtension();

‎apps/files/lib/Service/LivePhotosService.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,22 @@ public function getLivePhotoPeerId(int $fileId): ?int {
3333

3434
return (int)$metadata->getString('files-live-photo');
3535
}
36+
37+
/**
38+
* Get the associated live photo for multiple file ids
39+
* @param int[] $fileIds
40+
* @return int[]
41+
*/
42+
public function getLivePhotoPeerIds(array $fileIds): array {
43+
$metadata = $this->filesMetadataManager->getMetadataForFiles($fileIds);
44+
$peersIds = [];
45+
foreach ($metadata as $item) {
46+
if (!$item->hasKey('files-live-photo')) {
47+
continue;
48+
}
49+
50+
$peersIds[] = (int)$item->getString('files-live-photo');
51+
}
52+
return $peersIds;
53+
}
3654
}

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@
395395
'OCP\\Files\\AlreadyExistsException' => $baseDir . '/lib/public/Files/AlreadyExistsException.php',
396396
'OCP\\Files\\AppData\\IAppDataFactory' => $baseDir . '/lib/public/Files/AppData/IAppDataFactory.php',
397397
'OCP\\Files\\Cache\\AbstractCacheEvent' => $baseDir . '/lib/public/Files/Cache/AbstractCacheEvent.php',
398+
'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php',
398399
'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php',
399400
'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php',
400401
'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
436436
'OCP\\Files\\AlreadyExistsException' => __DIR__ . '/../../..' . '/lib/public/Files/AlreadyExistsException.php',
437437
'OCP\\Files\\AppData\\IAppDataFactory' => __DIR__ . '/../../..' . '/lib/public/Files/AppData/IAppDataFactory.php',
438438
'OCP\\Files\\Cache\\AbstractCacheEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/AbstractCacheEvent.php',
439+
'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php',
439440
'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php',
440441
'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php',
441442
'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php',

‎lib/private/Files/Cache/Cache.php‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use OC\SystemConfig;
1717
use OCP\DB\QueryBuilder\IQueryBuilder;
1818
use OCP\EventDispatcher\IEventDispatcher;
19+
use OCP\Files\Cache\CacheEntriesRemovedEvent;
1920
use OCP\Files\Cache\CacheEntryInsertedEvent;
2021
use OCP\Files\Cache\CacheEntryRemovedEvent;
2122
use OCP\Files\Cache\CacheEntryUpdatedEvent;
@@ -611,15 +612,19 @@ private function removeChildren(ICacheEntry $entry) {
611612
$query->executeStatement();
612613
}
613614

615+
$cacheEntryRemovedEvents = [];
614616
foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
615617
$cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
616618
$this->storage,
617619
$filePath,
618620
$fileId,
619621
$this->getNumericStorageId()
620622
);
623+
$cacheEntryRemovedEvents[] = $cacheEntryRemovedEvent;
621624
$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
622625
}
626+
$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents));
627+
623628
}
624629

625630
/**
@@ -775,7 +780,10 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
775780
$this->connection->commit();
776781

777782
if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
778-
$this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
783+
$event = new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId());
784+
$this->eventDispatcher->dispatchTyped($event);
785+
$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent([$event]));
786+
779787
$event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
780788
$this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
781789
$this->eventDispatcher->dispatchTyped($event);

‎lib/private/FilesMetadata/FilesMetadataManager.php‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use OCP\DB\Exception as DBException;
2121
use OCP\DB\QueryBuilder\IQueryBuilder;
2222
use OCP\EventDispatcher\IEventDispatcher;
23-
use OCP\Files\Cache\CacheEntryRemovedEvent;
23+
use OCP\Files\Cache\CacheEntriesRemovedEvent;
2424
use OCP\Files\Events\Node\NodeWrittenEvent;
2525
use OCP\Files\InvalidPathException;
2626
use OCP\Files\Node;
@@ -214,6 +214,20 @@ public function deleteMetadata(int $fileId): void {
214214
}
215215
}
216216

217+
public function deleteMetadataForFiles(array $fileIds): void {
218+
try {
219+
$this->metadataRequestService->dropMetadataForFiles($fileIds);
220+
} catch (Exception $e) {
221+
$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
222+
}
223+
224+
try {
225+
$this->indexRequestService->dropIndexForFiles($fileIds);
226+
} catch (Exception $e) {
227+
$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
228+
}
229+
}
230+
217231
/**
218232
* @param IQueryBuilder $qb
219233
* @param string $fileTableAlias alias of the table that contains data about files
@@ -301,6 +315,6 @@ public function initMetadata(
301315
*/
302316
public static function loadListeners(IEventDispatcher $eventDispatcher): void {
303317
$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
304-
$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
318+
$eventDispatcher->addServiceListener(CacheEntriesRemovedEvent::class, MetadataDelete::class);
305319
}
306320
}

‎lib/private/FilesMetadata/Listener/MetadataDelete.php‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@
1111
use Exception;
1212
use OCP\EventDispatcher\Event;
1313
use OCP\EventDispatcher\IEventListener;
14-
use OCP\Files\Cache\CacheEntryRemovedEvent;
14+
use OCP\Files\Cache\CacheEntriesRemovedEvent;
1515
use OCP\FilesMetadata\IFilesMetadataManager;
1616
use Psr\Log\LoggerInterface;
1717

1818
/**
1919
* Handle file deletion event and remove stored metadata related to the deleted file
2020
*
21-
* @template-implements IEventListener<CacheEntryRemovedEvent>
21+
* @template-implements IEventListener<CacheEntriesRemovedEvent>
2222
*/
2323
class MetadataDelete implements IEventListener {
2424
public function __construct(
@@ -28,15 +28,23 @@ public function __construct(
2828
}
2929

3030
public function handle(Event $event): void {
31-
if (!($event instanceof CacheEntryRemovedEvent)) {
31+
if (!($event instanceof CacheEntriesRemovedEvent)) {
3232
return;
3333
}
3434

35-
try {
36-
$nodeId = $event->getFileId();
37-
if ($nodeId > 0) {
38-
$this->filesMetadataManager->deleteMetadata($nodeId);
35+
$entries = $event->getCacheEntryRemovedEvents();
36+
$fileIds = [];
37+
38+
foreach ($entries as $entry) {
39+
try {
40+
$fileIds[] = $entry->getFileId();
41+
} catch (Exception $e) {
42+
$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
3943
}
44+
}
45+
46+
try {
47+
$this->filesMetadataManager->deleteMetadataForFiles($fileIds);
4048
} catch (Exception $e) {
4149
$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
4250
}

‎lib/private/FilesMetadata/Service/IndexRequestService.php‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,4 +175,30 @@ public function dropIndex(int $fileId, string $key = ''): void {
175175

176176
$qb->executeStatement();
177177
}
178+
179+
/**
180+
* Drop indexes related to multiple file ids
181+
* if a key is specified, only drop entries related to it
182+
*
183+
* @param int[] $fileIds file ids
184+
* @param string $key metadata key
185+
*
186+
* @throws DbException
187+
*/
188+
public function dropIndexForFiles(array $fileIds, string $key = ''): void {
189+
$chunks = array_chunk($fileIds, 1000);
190+
191+
foreach ($chunks as $chunk) {
192+
$qb = $this->dbConnection->getQueryBuilder();
193+
$expr = $qb->expr();
194+
$qb->delete(self::TABLE_METADATA_INDEX)
195+
->where($expr->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
196+
197+
if ($key !== '') {
198+
$qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key)));
199+
}
200+
201+
$qb->executeStatement();
202+
}
203+
}
178204
}

‎lib/private/FilesMetadata/Service/MetadataRequestService.php‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,10 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata {
108108
*/
109109
public function getMetadataFromFileIds(array $fileIds): array {
110110
$qb = $this->dbConnection->getQueryBuilder();
111-
$qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA);
112-
$qb->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
111+
$qb->select('file_id', 'json', 'sync_token')
112+
->from(self::TABLE_METADATA)
113+
->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
114+
->runAcrossAllShards();
113115

114116
$list = [];
115117
$result = $qb->executeQuery();
@@ -143,6 +145,22 @@ public function dropMetadata(int $fileId): void {
143145
$qb->executeStatement();
144146
}
145147

148+
/**
149+
* @param int[] $fileIds
150+
* @return void
151+
* @throws Exception
152+
*/
153+
public function dropMetadataForFiles(array $fileIds): void {
154+
$chunks = array_chunk($fileIds, 1000);
155+
156+
foreach ($chunks as $chunk) {
157+
$qb = $this->dbConnection->getQueryBuilder();
158+
$qb->delete(self::TABLE_METADATA)
159+
->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
160+
$qb->executeStatement();
161+
}
162+
}
163+
146164
/**
147165
* update metadata in the database
148166
*

0 commit comments

Comments
 (0)