[stable31] allow configuring multiple object store backends

apps/files/appinfo/info.xml

@@ -53,6 +53,8 @@
 		<command>OCA\Files\Command\Object\Info</command>
 		<command>OCA\Files\Command\Object\ListObject</command>
 		<command>OCA\Files\Command\Object\Orphans</command>
+		<command>OCA\Files\Command\Object\Multi\Users</command>
+		<command>OCA\Files\Command\Object\Multi\Rename</command>
 		<command>OCA\Files\Command\WindowsCompatibleFilenames</command>
 	</commands>
 

apps/files/composer/composer/autoload_classmap.php

@@ -36,6 +36,8 @@
     'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
     'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
     'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
+    'OCA\\Files\\Command\\Object\\Multi\\Rename' => $baseDir . '/../lib/Command/Object/Multi/Rename.php',
+    'OCA\\Files\\Command\\Object\\Multi\\Users' => $baseDir . '/../lib/Command/Object/Multi/Users.php',
     'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
     'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
     'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',

apps/files/composer/composer/autoload_static.php

@@ -51,6 +51,8 @@ class ComposerStaticInitFiles
         'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
         'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
         'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
+        'OCA\\Files\\Command\\Object\\Multi\\Rename' => __DIR__ . '/..' . '/../lib/Command/Object/Multi/Rename.php',
+        'OCA\\Files\\Command\\Object\\Multi\\Users' => __DIR__ . '/..' . '/../lib/Command/Object/Multi/Users.php',
         'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
         'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
         'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',

apps/files/lib/Command/Object/Multi/Rename.php

@@ -0,0 +1,108 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Command\Object\Multi;
+
+use OC\Core\Command\Base;
+use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Helper\QuestionHelper;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
+
+class Rename extends Base {
+	public function __construct(
+		private readonly IDBConnection $connection,
+		private readonly PrimaryObjectStoreConfig $objectStoreConfig,
+		private readonly IConfig $config,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		parent::configure();
+		$this
+			->setName('files:object:multi:rename-config')
+			->setDescription('Rename an object store configuration and move all users over to the new configuration,')
+			->addArgument('source', InputArgument::REQUIRED, 'Object store configuration to rename')
+			->addArgument('target', InputArgument::REQUIRED, 'New name for the object store configuration');
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		$source = $input->getArgument('source');
+		$target = $input->getArgument('target');
+
+		$configs = $this->objectStoreConfig->getObjectStoreConfigs();
+		if (!isset($configs[$source])) {
+			$output->writeln('<error>Unknown object store configuration: ' . $source . '</error>');
+			return 1;
+		}
+
+		if ($source === 'root') {
+			$output->writeln('<error>Renaming the root configuration is not supported.</error>');
+			return 1;
+		}
+
+		if ($source === 'default') {
+			$output->writeln('<error>Renaming the default configuration is not supported.</error>');
+			return 1;
+		}
+
+		if (!isset($configs[$target])) {
+			$output->writeln('<comment>Target object store configuration ' . $target . ' doesn\'t exist yet.</comment>');
+			$output->writeln('The target configuration can be created automatically.');
+			$output->writeln('However, as this depends on modifying the config.php, this only works as long as the instance runs on a single node or all nodes in a clustered setup have a shared config file (such as from a shared network mount).');
+			$output->writeln('If the different nodes have a separate copy of the config.php file, the automatic object store configuration creation will lead to the configuration going out of sync.');
+			$output->writeln('If these requirements are not met, you can manually create the target object store configuration in each node\'s configuration before running the command.');
+			$output->writeln('');
+			$output->writeln('<error>Failure to check these requirements will lead to data loss for users.</error>');
+
+			/** @var QuestionHelper $helper */
+			$helper = $this->getHelper('question');
+			$question = new ConfirmationQuestion('Automatically create target object store configuration? [y/N] ', false);
+			if ($helper->ask($input, $output, $question)) {
+				$configs[$target] = $configs[$source];
+
+				// update all aliases
+				foreach ($configs as &$config) {
+					if ($config === $source) {
+						$config = $target;
+					}
+				}
+				$this->config->setSystemValue('objectstore', $configs);
+			} else {
+				return 0;
+			}
+		} elseif (($configs[$source] !== $configs[$target]) || $configs[$source] !== $target) {
+			$output->writeln('<error>Source and target configuration differ.</error>');
+			$output->writeln('');
+			$output->writeln('To ensure proper migration of users, the source and target configuration must be the same to ensure that the objects for the moved users exist on the target configuration.');
+			$output->writeln('The usual migration process consists of creating a clone of the old configuration, moving the users from the old configuration to the new one, and then adjust the old configuration that is longer used.');
+			return 1;
+		}
+
+		$query = $this->connection->getQueryBuilder();
+		$query->update('preferences')
+			->set('configvalue', $query->createNamedParameter($target))
+			->where($query->expr()->eq('appid', $query->createNamedParameter('homeobjectstore')))
+			->andWhere($query->expr()->eq('configkey', $query->createNamedParameter('objectstore')))
+			->andWhere($query->expr()->eq('configvalue', $query->createNamedParameter($source)));
+		$count = $query->executeStatement();
+
+		if ($count > 0) {
+			$output->writeln('Moved <info>' . $count . '</info> users');
+		} else {
+			$output->writeln('No users moved');
+		}
+
+		return 0;
+	}
+}

apps/files/lib/Command/Object/Multi/Users.php

@@ -0,0 +1,98 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Files\Command\Object\Multi;
+
+use OC\Core\Command\Base;
+use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
+use OCP\IConfig;
+use OCP\IUser;
+use OCP\IUserManager;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Users extends Base {
+	public function __construct(
+		private readonly IUserManager $userManager,
+		private readonly PrimaryObjectStoreConfig $objectStoreConfig,
+		private readonly IConfig $config,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure(): void {
+		parent::configure();
+		$this
+			->setName('files:object:multi:users')
+			->setDescription('Get the mapping between users and object store buckets')
+			->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, 'Only list users using the specified bucket')
+			->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'Only list users using the specified object store configuration')
+			->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Only show the mapping for the specified user, ignores all other options');
+	}
+
+	public function execute(InputInterface $input, OutputInterface $output): int {
+		if ($userId = $input->getOption('user')) {
+			$user = $this->userManager->get($userId);
+			if (!$user) {
+				$output->writeln("<error>User $userId not found</error>");
+				return 1;
+			}
+			$users = new \ArrayIterator([$user]);
+		} else {
+			$bucket = (string)$input->getOption('bucket');
+			$objectStore = (string)$input->getOption('object-store');
+			if ($bucket !== '' && $objectStore === '') {
+				$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket));
+			} elseif ($bucket === '' && $objectStore !== '') {
+				$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore));
+			} elseif ($bucket) {
+				$users = $this->getUsers(array_intersect(
+					$this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket),
+					$this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore)
+				));
+			} else {
+				$users = $this->userManager->getSeenUsers();
+			}
+		}
+
+		$this->writeStreamingTableInOutputFormat($input, $output, $this->infoForUsers($users), 100);
+		return 0;
+	}
+
+	/**
+	 * @param string[] $userIds
+	 * @return \Iterator<IUser>
+	 */
+	private function getUsers(array $userIds): \Iterator {
+		foreach ($userIds as $userId) {
+			$user = $this->userManager->get($userId);
+			if ($user) {
+				yield $user;
+			}
+		}
+	}
+
+	/**
+	 * @param \Iterator<IUser> $users
+	 * @return \Iterator<array>
+	 */
+	private function infoForUsers(\Iterator $users): \Iterator {
+		foreach ($users as $user) {
+			yield $this->infoForUser($user);
+		}
+	}
+
+	private function infoForUser(IUser $user): array {
+		return [
+			'user' => $user->getUID(),
+			'object-store' => $this->objectStoreConfig->getObjectStoreForUser($user),
+			'bucket' => $this->objectStoreConfig->getSetBucketForUser($user) ?? 'unset',
+		];
+	}
+}

lib/composer/composer/autoload_classmap.php

@@ -1645,6 +1645,7 @@
     'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
     'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
     'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
+    'OC\\Files\\ObjectStore\\InvalidObjectStoreConfigurationException' => $baseDir . '/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php',
     'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
     'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
     'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

lib/composer/composer/autoload_static.php

@@ -1694,6 +1694,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
         'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
         'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
+        'OC\\Files\\ObjectStore\\InvalidObjectStoreConfigurationException' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php',
         'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
         'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
         'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OC\Files\ObjectStore;
+
+class InvalidObjectStoreConfigurationException extends \Exception {
+
+}

lib/private/Files/ObjectStore/Mapper.php

@@ -7,7 +7,6 @@
  */
 namespace OC\Files\ObjectStore;
 
-use OCP\IConfig;
 use OCP\IUser;
 
 /**
@@ -18,33 +17,17 @@
  * Map a user to a bucket.
  */
 class Mapper {
-	/** @var IUser */
-	private $user;
-
-	/** @var IConfig */
-	private $config;
-
-	/**
-	 * Mapper constructor.
-	 *
-	 * @param IUser $user
-	 * @param IConfig $config
-	 */
-	public function __construct(IUser $user, IConfig $config) {
-		$this->user = $user;
-		$this->config = $config;
+	public function __construct(
+		private readonly IUser $user,
+		private readonly array $config,
+	) {
 	}
 
-	/**
-	 * @param int $numBuckets
-	 * @return string
-	 */
-	public function getBucket($numBuckets = 64) {
+	public function getBucket(int $numBuckets = 64): string {
 		// Get the bucket config and shift if provided.
 		// Allow us to prevent writing in old filled buckets
-		$config = $this->config->getSystemValue('objectstore_multibucket');
-		$minBucket = is_array($config) && isset($config['arguments']['min_bucket'])
-			? (int)$config['arguments']['min_bucket']
+		$minBucket = isset($this->config['arguments']['min_bucket'])
+			? (int)$this->config['arguments']['min_bucket']
 			: 0;
 
 		$hash = md5($this->user->getUID());