Skip to content

Commit 6b02bb1

Browse files
authored
Merge pull request #56557 from nextcloud/backport/52786/stable31
[stable31] allow configuring multiple object store backends
2 parents 1374e07 + d1a760d commit 6b02bb1

File tree

13 files changed

+646
-59
lines changed

13 files changed

+646
-59
lines changed

apps/files/appinfo/info.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
<command>OCA\Files\Command\Object\Info</command>
5454
<command>OCA\Files\Command\Object\ListObject</command>
5555
<command>OCA\Files\Command\Object\Orphans</command>
56+
<command>OCA\Files\Command\Object\Multi\Users</command>
57+
<command>OCA\Files\Command\Object\Multi\Rename</command>
5658
<command>OCA\Files\Command\WindowsCompatibleFilenames</command>
5759
</commands>
5860

apps/files/composer/composer/autoload_classmap.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
3737
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
3838
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
39+
'OCA\\Files\\Command\\Object\\Multi\\Rename' => $baseDir . '/../lib/Command/Object/Multi/Rename.php',
40+
'OCA\\Files\\Command\\Object\\Multi\\Users' => $baseDir . '/../lib/Command/Object/Multi/Users.php',
3941
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
4042
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
4143
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',

apps/files/composer/composer/autoload_static.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ class ComposerStaticInitFiles
5151
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
5252
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
5353
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
54+
'OCA\\Files\\Command\\Object\\Multi\\Rename' => __DIR__ . '/..' . '/../lib/Command/Object/Multi/Rename.php',
55+
'OCA\\Files\\Command\\Object\\Multi\\Users' => __DIR__ . '/..' . '/../lib/Command/Object/Multi/Users.php',
5456
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
5557
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
5658
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object\Multi;
10+
11+
use OC\Core\Command\Base;
12+
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
13+
use OCP\IConfig;
14+
use OCP\IDBConnection;
15+
use Symfony\Component\Console\Helper\QuestionHelper;
16+
use Symfony\Component\Console\Input\InputArgument;
17+
use Symfony\Component\Console\Input\InputInterface;
18+
use Symfony\Component\Console\Output\OutputInterface;
19+
use Symfony\Component\Console\Question\ConfirmationQuestion;
20+
21+
class Rename extends Base {
22+
public function __construct(
23+
private readonly IDBConnection $connection,
24+
private readonly PrimaryObjectStoreConfig $objectStoreConfig,
25+
private readonly IConfig $config,
26+
) {
27+
parent::__construct();
28+
}
29+
30+
protected function configure(): void {
31+
parent::configure();
32+
$this
33+
->setName('files:object:multi:rename-config')
34+
->setDescription('Rename an object store configuration and move all users over to the new configuration,')
35+
->addArgument('source', InputArgument::REQUIRED, 'Object store configuration to rename')
36+
->addArgument('target', InputArgument::REQUIRED, 'New name for the object store configuration');
37+
}
38+
39+
public function execute(InputInterface $input, OutputInterface $output): int {
40+
$source = $input->getArgument('source');
41+
$target = $input->getArgument('target');
42+
43+
$configs = $this->objectStoreConfig->getObjectStoreConfigs();
44+
if (!isset($configs[$source])) {
45+
$output->writeln('<error>Unknown object store configuration: ' . $source . '</error>');
46+
return 1;
47+
}
48+
49+
if ($source === 'root') {
50+
$output->writeln('<error>Renaming the root configuration is not supported.</error>');
51+
return 1;
52+
}
53+
54+
if ($source === 'default') {
55+
$output->writeln('<error>Renaming the default configuration is not supported.</error>');
56+
return 1;
57+
}
58+
59+
if (!isset($configs[$target])) {
60+
$output->writeln('<comment>Target object store configuration ' . $target . ' doesn\'t exist yet.</comment>');
61+
$output->writeln('The target configuration can be created automatically.');
62+
$output->writeln('However, as this depends on modifying the config.php, this only works as long as the instance runs on a single node or all nodes in a clustered setup have a shared config file (such as from a shared network mount).');
63+
$output->writeln('If the different nodes have a separate copy of the config.php file, the automatic object store configuration creation will lead to the configuration going out of sync.');
64+
$output->writeln('If these requirements are not met, you can manually create the target object store configuration in each node\'s configuration before running the command.');
65+
$output->writeln('');
66+
$output->writeln('<error>Failure to check these requirements will lead to data loss for users.</error>');
67+
68+
/** @var QuestionHelper $helper */
69+
$helper = $this->getHelper('question');
70+
$question = new ConfirmationQuestion('Automatically create target object store configuration? [y/N] ', false);
71+
if ($helper->ask($input, $output, $question)) {
72+
$configs[$target] = $configs[$source];
73+
74+
// update all aliases
75+
foreach ($configs as &$config) {
76+
if ($config === $source) {
77+
$config = $target;
78+
}
79+
}
80+
$this->config->setSystemValue('objectstore', $configs);
81+
} else {
82+
return 0;
83+
}
84+
} elseif (($configs[$source] !== $configs[$target]) || $configs[$source] !== $target) {
85+
$output->writeln('<error>Source and target configuration differ.</error>');
86+
$output->writeln('');
87+
$output->writeln('To ensure proper migration of users, the source and target configuration must be the same to ensure that the objects for the moved users exist on the target configuration.');
88+
$output->writeln('The usual migration process consists of creating a clone of the old configuration, moving the users from the old configuration to the new one, and then adjust the old configuration that is longer used.');
89+
return 1;
90+
}
91+
92+
$query = $this->connection->getQueryBuilder();
93+
$query->update('preferences')
94+
->set('configvalue', $query->createNamedParameter($target))
95+
->where($query->expr()->eq('appid', $query->createNamedParameter('homeobjectstore')))
96+
->andWhere($query->expr()->eq('configkey', $query->createNamedParameter('objectstore')))
97+
->andWhere($query->expr()->eq('configvalue', $query->createNamedParameter($source)));
98+
$count = $query->executeStatement();
99+
100+
if ($count > 0) {
101+
$output->writeln('Moved <info>' . $count . '</info> users');
102+
} else {
103+
$output->writeln('No users moved');
104+
}
105+
106+
return 0;
107+
}
108+
}
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object\Multi;
10+
11+
use OC\Core\Command\Base;
12+
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
13+
use OCP\IConfig;
14+
use OCP\IUser;
15+
use OCP\IUserManager;
16+
use Symfony\Component\Console\Input\InputInterface;
17+
use Symfony\Component\Console\Input\InputOption;
18+
use Symfony\Component\Console\Output\OutputInterface;
19+
20+
class Users extends Base {
21+
public function __construct(
22+
private readonly IUserManager $userManager,
23+
private readonly PrimaryObjectStoreConfig $objectStoreConfig,
24+
private readonly IConfig $config,
25+
) {
26+
parent::__construct();
27+
}
28+
29+
protected function configure(): void {
30+
parent::configure();
31+
$this
32+
->setName('files:object:multi:users')
33+
->setDescription('Get the mapping between users and object store buckets')
34+
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, 'Only list users using the specified bucket')
35+
->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'Only list users using the specified object store configuration')
36+
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Only show the mapping for the specified user, ignores all other options');
37+
}
38+
39+
public function execute(InputInterface $input, OutputInterface $output): int {
40+
if ($userId = $input->getOption('user')) {
41+
$user = $this->userManager->get($userId);
42+
if (!$user) {
43+
$output->writeln("<error>User $userId not found</error>");
44+
return 1;
45+
}
46+
$users = new \ArrayIterator([$user]);
47+
} else {
48+
$bucket = (string)$input->getOption('bucket');
49+
$objectStore = (string)$input->getOption('object-store');
50+
if ($bucket !== '' && $objectStore === '') {
51+
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket));
52+
} elseif ($bucket === '' && $objectStore !== '') {
53+
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore));
54+
} elseif ($bucket) {
55+
$users = $this->getUsers(array_intersect(
56+
$this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket),
57+
$this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore)
58+
));
59+
} else {
60+
$users = $this->userManager->getSeenUsers();
61+
}
62+
}
63+
64+
$this->writeStreamingTableInOutputFormat($input, $output, $this->infoForUsers($users), 100);
65+
return 0;
66+
}
67+
68+
/**
69+
* @param string[] $userIds
70+
* @return \Iterator<IUser>
71+
*/
72+
private function getUsers(array $userIds): \Iterator {
73+
foreach ($userIds as $userId) {
74+
$user = $this->userManager->get($userId);
75+
if ($user) {
76+
yield $user;
77+
}
78+
}
79+
}
80+
81+
/**
82+
* @param \Iterator<IUser> $users
83+
* @return \Iterator<array>
84+
*/
85+
private function infoForUsers(\Iterator $users): \Iterator {
86+
foreach ($users as $user) {
87+
yield $this->infoForUser($user);
88+
}
89+
}
90+
91+
private function infoForUser(IUser $user): array {
92+
return [
93+
'user' => $user->getUID(),
94+
'object-store' => $this->objectStoreConfig->getObjectStoreForUser($user),
95+
'bucket' => $this->objectStoreConfig->getSetBucketForUser($user) ?? 'unset',
96+
];
97+
}
98+
}

lib/composer/composer/autoload_classmap.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,6 +1645,7 @@
16451645
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
16461646
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
16471647
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
1648+
'OC\\Files\\ObjectStore\\InvalidObjectStoreConfigurationException' => $baseDir . '/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php',
16481649
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
16491650
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
16501651
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

lib/composer/composer/autoload_static.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1694,6 +1694,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
16941694
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
16951695
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
16961696
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
1697+
'OC\\Files\\ObjectStore\\InvalidObjectStoreConfigurationException' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/InvalidObjectStoreConfigurationException.php',
16971698
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
16981699
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
16991700
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2025 Robin Appelman <[email protected]>
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OC\Files\ObjectStore;
10+
11+
class InvalidObjectStoreConfigurationException extends \Exception {
12+
13+
}

lib/private/Files/ObjectStore/Mapper.php

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
*/
88
namespace OC\Files\ObjectStore;
99

10-
use OCP\IConfig;
1110
use OCP\IUser;
1211

1312
/**
@@ -18,33 +17,17 @@
1817
* Map a user to a bucket.
1918
*/
2019
class Mapper {
21-
/** @var IUser */
22-
private $user;
23-
24-
/** @var IConfig */
25-
private $config;
26-
27-
/**
28-
* Mapper constructor.
29-
*
30-
* @param IUser $user
31-
* @param IConfig $config
32-
*/
33-
public function __construct(IUser $user, IConfig $config) {
34-
$this->user = $user;
35-
$this->config = $config;
20+
public function __construct(
21+
private readonly IUser $user,
22+
private readonly array $config,
23+
) {
3624
}
3725

38-
/**
39-
* @param int $numBuckets
40-
* @return string
41-
*/
42-
public function getBucket($numBuckets = 64) {
26+
public function getBucket(int $numBuckets = 64): string {
4327
// Get the bucket config and shift if provided.
4428
// Allow us to prevent writing in old filled buckets
45-
$config = $this->config->getSystemValue('objectstore_multibucket');
46-
$minBucket = is_array($config) && isset($config['arguments']['min_bucket'])
47-
? (int)$config['arguments']['min_bucket']
29+
$minBucket = isset($this->config['arguments']['min_bucket'])
30+
? (int)$this->config['arguments']['min_bucket']
4831
: 0;
4932

5033
$hash = md5($this->user->getUID());

0 commit comments

Comments
 (0)