Merge pull request #56395 from nextcloud/carl/cache-bucket-exists

perf(s3): Cache whether bucket exists

Author: AndyScherzinger

lib/private/Files/ObjectStore/S3ConnectionTrait.php

@@ -15,6 +15,8 @@
 use GuzzleHttp\Promise\Create;
 use GuzzleHttp\Promise\RejectedPromise;
 use OCP\Files\StorageNotAvailableException;
+use OCP\ICache;
+use OCP\ICacheFactory;
 use OCP\ICertificateManager;
 use OCP\Server;
 use Psr\Log\LoggerInterface;
@@ -28,6 +30,8 @@ trait S3ConnectionTrait {
 
 	protected ?S3Client $connection = null;
 
+	private ?ICache $existingBucketsCache = null;
+
 	protected function parseParams($params) {
 		if (empty($params['bucket'])) {
 			throw new \Exception('Bucket has to be configured.');
@@ -82,6 +86,11 @@ public function getConnection() {
 			return $this->connection;
 		}
 
+		if ($this->existingBucketsCache === null) {
+			$this->existingBucketsCache = Server::get(ICacheFactory::class)
+				->createLocal('s3-bucket-exists-cache');
+		}
+
 		$scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
 		$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
 
@@ -143,22 +152,30 @@ public function getConnection() {
 					['app' => 'objectstore']);
 			}
 
-			if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
-				try {
-					$logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
-					if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
-						throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket);
-					}
-					$this->connection->createBucket(['Bucket' => $this->bucket]);
-					$this->testTimeout();
-				} catch (S3Exception $e) {
-					$logger->debug('Invalid remote storage.', [
-						'exception' => $e,
-						'app' => 'objectstore',
-					]);
-					if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') {
-						throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
+			if ($this->params['verify_bucket_exists']) {
+				$cacheKey = $this->params['hostname'] . $this->bucket;
+				$exist = $this->existingBucketsCache->get($cacheKey) === 1;
+
+				if (!$exist) {
+					if (!$this->connection->doesBucketExist($this->bucket)) {
+						try {
+							$logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
+							if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
+								throw new StorageNotAvailableException('The bucket will not be created because the name is not dns compatible, please correct it: ' . $this->bucket);
+							}
+							$this->connection->createBucket(['Bucket' => $this->bucket]);
+							$this->testTimeout();
+						} catch (S3Exception $e) {
+							$logger->debug('Invalid remote storage.', [
+								'exception' => $e,
+								'app' => 'objectstore',
+							]);
+							if ($e->getAwsErrorCode() !== 'BucketAlreadyOwnedByYou') {
+								throw new StorageNotAvailableException('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
+							}
+						}
 					}
+					$this->existingBucketsCache->set($cacheKey, 1);
 				}
 			}