From 8cb63c4b6ad79bf87067aca09fd2bade326a8702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=B4me=20Chilliet?= Date: Thu, 28 Apr 2022 14:07:08 +0200 Subject: [PATCH 01/22] Implement getExportEstimatedSize in FilesMigrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Côme Chilliet --- lib/Migrator/FilesMigrator.php | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/Migrator/FilesMigrator.php b/lib/Migrator/FilesMigrator.php index befe7c47..3a571834 100644 --- a/lib/Migrator/FilesMigrator.php +++ b/lib/Migrator/FilesMigrator.php @@ -86,6 +86,35 @@ public function __construct( $this->l10n = $l10n; } + /** + * {@inheritDoc} + */ + public function getExportEstimatedSize(IUser $user): int { + $uid = $user->getUID(); + + $userFolder = $this->root->getUserFolder($uid); + + $size = $userFolder->getSize() / 1024; + + try { + $versionsFolder = $this->root->get('/'.$uid.'/'.FilesVersionsStorage::VERSIONS_ROOT); + if ($versionsFolder instanceof Folder) { + return 0; + } + $size += $versionsFolder->getSize() / 1024; + } catch (\Throwable $e) { + return 0; + } + + // 1MiB for tags and system tags + $size += 1024; + + // 2MiB for comments + $size += 2048; + + return (int)ceil($size); + } + /** * {@inheritDoc} */ From bef3771085f136820549a9bc513c9151ad702b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=B4me=20Chilliet?= Date: Thu, 28 Apr 2022 14:29:30 +0200 Subject: [PATCH 02/22] Add estimate export size to service and controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Côme Chilliet --- appinfo/routes.php | 1 + lib/Controller/ApiController.php | 75 ++++++++++++++++++---------- lib/Service/UserMigrationService.php | 18 +++++++ 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index aa84a70a..149e568e 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -33,6 +33,7 @@ ['name' => 'Api#migrators', 'url' => '/api/v{apiVersion}/migrators', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#status', 'url' => '/api/v{apiVersion}/status', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#cancel', 'url' => '/api/v{apiVersion}/cancel', 'verb' => 'PUT', 'requirements' => $requirements], + ['name' => 'Api#estimateExportSize', 'url' => '/api/v{apiVersion}/estimate', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'Api#export', 'url' => '/api/v{apiVersion}/export', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'Api#import', 'url' => '/api/v{apiVersion}/import', 'verb' => 'POST', 'requirements' => $requirements], ], diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 3c7c907e..7120fee4 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -170,20 +170,38 @@ public function cancel(): DataResponse { return new DataResponse([], Http::STATUS_OK); } - /** - * @NoAdminRequired - * @NoSubAdminRequired - * @PasswordConfirmationRequired - * + /* * @throws OCSException */ - public function export(array $migrators): DataResponse { + private function checkJobAndGetUser(): IUser { $user = $this->userSession->getUser(); if (empty($user)) { throw new OCSException('No user currently logged in'); } + try { + $job = $this->migrationService->getCurrentJob($user); + } catch (UserMigrationException $e) { + throw new OCSException('Error getting current user migration operation'); + } + + if (!empty($job)) { + throw new OCSException('User migration operation already queued'); + } + + return $user; + } + + /** + * @NoAdminRequired + * @NoSubAdminRequired + * + * @throws OCSException + */ + public function estimateExportSize(array $migrators): DataResponse { + $user = $this->checkJobAndGetUser(); + /** @var string[] $availableMigrators */ $availableMigrators = array_map( fn (IMigrator $migrator) => $migrator->getId(), @@ -197,13 +215,34 @@ public function export(array $migrators): DataResponse { } try { - $job = $this->migrationService->getCurrentJob($user); + $size = $this->migrationService->estimateExportSize($user, $migrators); } catch (UserMigrationException $e) { - throw new OCSException('Error getting current user migration operation'); + throw new OCSException('Error estimating export size'); } - if (!empty($job)) { - throw new OCSException('User migration operation already queued'); + return new DataResponse(['kibibytes' => $size], Http::STATUS_OK); + } + + /** + * @NoAdminRequired + * @NoSubAdminRequired + * @PasswordConfirmationRequired + * + * @throws OCSException + */ + public function export(array $migrators): DataResponse { + $user = $this->checkJobAndGetUser(); + + /** @var string[] $availableMigrators */ + $availableMigrators = array_map( + fn (IMigrator $migrator) => $migrator->getId(), + $this->migrationService->getMigrators(), + ); + + foreach ($migrators as $migrator) { + if (!in_array($migrator, $availableMigrators, true)) { + throw new OCSException("Requested migrator \"$migrator\" not available"); + } } try { @@ -223,24 +262,10 @@ public function export(array $migrators): DataResponse { * @throws OCSException */ public function import(string $path): DataResponse { - $author = $this->userSession->getUser(); + $author = $this->checkJobAndGetUser(); // Set target user to the author as importing into another user's account is not allowed for now $targetUser = $author; - if (empty($author)) { - throw new OCSException('No user currently logged in'); - } - - try { - $job = $this->migrationService->getCurrentJob($targetUser); - } catch (UserMigrationException $e) { - throw new OCSException('Error getting current user migration operation'); - } - - if (!empty($job)) { - throw new OCSException('User migration operation already queued'); - } - try { $this->migrationService->queueImportJob($author, $targetUser, $path); } catch (UserMigrationException $e) { diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index f80d08d4..777431c6 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -98,6 +98,24 @@ public function __construct( $this->mandatory = true; } + /** + * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. + * @return int Estimated size in KiB + */ + public function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { + $size = 1024; + + foreach ($this->getMigrators() as $migrator) { + if ($filteredMigratorList !== null && !in_array($migrator->getId(), $filteredMigratorList)) { + continue; + } + // TODO Cache this (so that user can check/uncheck migrators to see the difference in export size) + $size += $migrator->getExportEstimatedSize($user); + } + + return $size; + } + /** * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. * @throws UserMigrationException From e7946e5bacc6f3ca98285a3c7ac6cf768d1c6d80 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 03:27:42 +0000 Subject: [PATCH 03/22] Update method name Signed-off-by: Christopher Ng --- lib/Migrator/FilesMigrator.php | 2 +- lib/Service/UserMigrationService.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Migrator/FilesMigrator.php b/lib/Migrator/FilesMigrator.php index 3a571834..e9116a48 100644 --- a/lib/Migrator/FilesMigrator.php +++ b/lib/Migrator/FilesMigrator.php @@ -89,7 +89,7 @@ public function __construct( /** * {@inheritDoc} */ - public function getExportEstimatedSize(IUser $user): int { + public function getEstimatedExportSize(IUser $user): int { $uid = $user->getUID(); $userFolder = $this->root->getUserFolder($uid); diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index 777431c6..4b9f983e 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -110,7 +110,7 @@ public function estimateExportSize(IUser $user, ?array $filteredMigratorList = n continue; } // TODO Cache this (so that user can check/uncheck migrators to see the difference in export size) - $size += $migrator->getExportEstimatedSize($user); + $size += $migrator->getEstimatedExportSize($user); } return $size; From e7d0badc450cfd252dd0a12a7b7470e2973f4124 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 03:56:11 +0000 Subject: [PATCH 04/22] Subtract size of export file from estimate Signed-off-by: Christopher Ng --- lib/Migrator/FilesMigrator.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/Migrator/FilesMigrator.php b/lib/Migrator/FilesMigrator.php index e9116a48..b5558ddc 100644 --- a/lib/Migrator/FilesMigrator.php +++ b/lib/Migrator/FilesMigrator.php @@ -96,6 +96,17 @@ public function getEstimatedExportSize(IUser $user): int { $size = $userFolder->getSize() / 1024; + try { + $exportFile = $userFolder->get(ExportDestination::EXPORT_FILENAME); + if (!($exportFile instanceof File)) { + throw new \InvalidArgumentException('User export is not a file'); + } + + $size -= $exportFile->getSize() / 1024; + } catch (NotFoundException $e) { + // No size subtraction needed if export file doesn't exist + } + try { $versionsFolder = $this->root->get('/'.$uid.'/'.FilesVersionsStorage::VERSIONS_ROOT); if ($versionsFolder instanceof Folder) { From 00c50a57e939eb9b965fb3e5a8c6dc813caecd5f Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 04:16:30 +0000 Subject: [PATCH 05/22] Cache size estimates Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index 4b9f983e..209ff979 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -28,6 +28,7 @@ namespace OCA\UserMigration\Service; use OC\AppFramework\Bootstrap\Coordinator; +use OC\Cache\CappedMemoryCache; use OCA\UserMigration\BackgroundJob\UserExportJob; use OCA\UserMigration\BackgroundJob\UserImportJob; use OCA\UserMigration\Db\UserExport; @@ -71,6 +72,8 @@ class UserMigrationService { protected IJobList $jobList; + protected CappedMemoryCache $internalCache; + protected const ENTITY_JOB_MAP = [ UserExport::class => UserExportJob::class, UserImport::class => UserImportJob::class, @@ -94,6 +97,7 @@ public function __construct( $this->exportMapper = $exportMapper; $this->importMapper = $importMapper; $this->jobList = $jobList; + $this->internalCache = new CappedMemoryCache(); $this->mandatory = true; } @@ -103,16 +107,26 @@ public function __construct( * @return int Estimated size in KiB */ public function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { + $cacheKey = $user->getUID(); + if ($filteredMigratorList !== null) { + $cacheKey .= '::' .json_encode($filteredMigratorList); + } + + if ($this->internalCache->hasKey($cacheKey)) { + return $this->internalCache->get($cacheKey); + } + + // 1MiB for base user data $size = 1024; foreach ($this->getMigrators() as $migrator) { if ($filteredMigratorList !== null && !in_array($migrator->getId(), $filteredMigratorList)) { continue; } - // TODO Cache this (so that user can check/uncheck migrators to see the difference in export size) $size += $migrator->getEstimatedExportSize($user); } + $this->internalCache->set($cacheKey, $size); return $size; } From c1a7a5b8a44c9a172a907e0af0cc4356e73fface Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 04:18:34 +0000 Subject: [PATCH 06/22] Fix DocBlock Signed-off-by: Christopher Ng --- lib/Controller/ApiController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 7120fee4..95cff5db 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -170,7 +170,7 @@ public function cancel(): DataResponse { return new DataResponse([], Http::STATUS_OK); } - /* + /** * @throws OCSException */ private function checkJobAndGetUser(): IUser { From 90b23d208ca44a08cd2b9311d4b95545b19cd11b Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 04:26:16 +0000 Subject: [PATCH 07/22] Fix spacing Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index 209ff979..cab9e7f8 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -109,7 +109,7 @@ public function __construct( public function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { $cacheKey = $user->getUID(); if ($filteredMigratorList !== null) { - $cacheKey .= '::' .json_encode($filteredMigratorList); + $cacheKey .= '::' . json_encode($filteredMigratorList); } if ($this->internalCache->hasKey($cacheKey)) { From a5cc9ce62726f651a6ce94da440d04a0e193baea Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 22:36:11 +0000 Subject: [PATCH 08/22] Add CappedMemoryCache stub Signed-off-by: Christopher Ng --- tests/stubs/stub.phpstub | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/stubs/stub.phpstub b/tests/stubs/stub.phpstub index e6f787ba..46e37d55 100644 --- a/tests/stubs/stub.phpstub +++ b/tests/stubs/stub.phpstub @@ -116,3 +116,23 @@ namespace OC\Hooks { interface Emitter { } } + +namespace OC\Cache { + + use OCP\ICache; + + class CappedMemoryCache implements ICache, \ArrayAccess { + public function __construct($capacity = 512); + public function hasKey($key): bool {} + public function get($key) {} + public function set($key, $value, $ttl = 0): bool {} + public function remove($key) {} + public function clear($prefix = '') {} + public function offsetExists($offset): bool {} + public function &offsetGet($offset) {} + public function offsetSet($offset, $value): void {} + public function offsetUnset($offset): void {} + public function getData() {} + public static function isAvailable(): bool {} + } +} From d9ec84e7f7a018e1ab77a928b524ddeae8a39976 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 29 Apr 2022 23:57:32 +0000 Subject: [PATCH 09/22] Add comment about export filesize subtraction Signed-off-by: Christopher Ng --- lib/Migrator/FilesMigrator.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Migrator/FilesMigrator.php b/lib/Migrator/FilesMigrator.php index b5558ddc..c831847d 100644 --- a/lib/Migrator/FilesMigrator.php +++ b/lib/Migrator/FilesMigrator.php @@ -96,6 +96,7 @@ public function getEstimatedExportSize(IUser $user): int { $size = $userFolder->getSize() / 1024; + // Export file itself is not exported so we subtract it if existing try { $exportFile = $userFolder->get(ExportDestination::EXPORT_FILENAME); if (!($exportFile instanceof File)) { From 5a9f98de07d874546263eb0aa55e7a6aa5425eee Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Sat, 30 Apr 2022 00:25:31 +0000 Subject: [PATCH 10/22] Add exportable endpoint Signed-off-by: Christopher Ng --- appinfo/routes.php | 2 +- lib/Controller/ApiController.php | 53 +++++++++++++------------- lib/Service/NotExportableException.php | 32 ++++++++++++++++ lib/Service/UserMigrationService.php | 34 ++++++++++++++++- 4 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 lib/Service/NotExportableException.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 149e568e..6c82e422 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -33,7 +33,7 @@ ['name' => 'Api#migrators', 'url' => '/api/v{apiVersion}/migrators', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#status', 'url' => '/api/v{apiVersion}/status', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#cancel', 'url' => '/api/v{apiVersion}/cancel', 'verb' => 'PUT', 'requirements' => $requirements], - ['name' => 'Api#estimateExportSize', 'url' => '/api/v{apiVersion}/estimate', 'verb' => 'POST', 'requirements' => $requirements], + ['name' => 'Api#exportable', 'url' => '/api/v{apiVersion}/exportable', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#export', 'url' => '/api/v{apiVersion}/export', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'Api#import', 'url' => '/api/v{apiVersion}/import', 'verb' => 'POST', 'requirements' => $requirements], ], diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 95cff5db..6c9d7579 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -29,6 +29,7 @@ use OCA\UserMigration\AppInfo\Application; use OCA\UserMigration\Db\UserExport; use OCA\UserMigration\Db\UserImport; +use OCA\UserMigration\Service\NotExportableException; use OCA\UserMigration\Service\UserMigrationService; use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataResponse; @@ -170,6 +171,23 @@ public function cancel(): DataResponse { return new DataResponse([], Http::STATUS_OK); } + /** + * @throws OCSException + */ + private function checkMigrators(array $migrators): void { + /** @var string[] $availableMigrators */ + $availableMigrators = array_map( + fn (IMigrator $migrator) => $migrator->getId(), + $this->migrationService->getMigrators(), + ); + + foreach ($migrators as $migrator) { + if (!in_array($migrator, $availableMigrators, true)) { + throw new OCSException("Requested migrator \"$migrator\" not available"); + } + } + } + /** * @throws OCSException */ @@ -199,28 +217,20 @@ private function checkJobAndGetUser(): IUser { * * @throws OCSException */ - public function estimateExportSize(array $migrators): DataResponse { + public function exportable(?array $migrators): DataResponse { $user = $this->checkJobAndGetUser(); - /** @var string[] $availableMigrators */ - $availableMigrators = array_map( - fn (IMigrator $migrator) => $migrator->getId(), - $this->migrationService->getMigrators(), - ); - - foreach ($migrators as $migrator) { - if (!in_array($migrator, $availableMigrators, true)) { - throw new OCSException("Requested migrator \"$migrator\" not available"); - } + if (!is_null($migrators)) { + $this->checkMigrators($migrators); } try { - $size = $this->migrationService->estimateExportSize($user, $migrators); - } catch (UserMigrationException $e) { - throw new OCSException('Error estimating export size'); + $this->migrationService->checkExportability($user, $migrators); + } catch (NotExportableException $e) { + throw new OCSException($e->getMessage()); } - return new DataResponse(['kibibytes' => $size], Http::STATUS_OK); + return new DataResponse([], Http::STATUS_OK); } /** @@ -232,18 +242,7 @@ public function estimateExportSize(array $migrators): DataResponse { */ public function export(array $migrators): DataResponse { $user = $this->checkJobAndGetUser(); - - /** @var string[] $availableMigrators */ - $availableMigrators = array_map( - fn (IMigrator $migrator) => $migrator->getId(), - $this->migrationService->getMigrators(), - ); - - foreach ($migrators as $migrator) { - if (!in_array($migrator, $availableMigrators, true)) { - throw new OCSException("Requested migrator \"$migrator\" not available"); - } - } + $this->checkMigrators($migrators); try { $this->migrationService->queueExportJob($user, $migrators); diff --git a/lib/Service/NotExportableException.php b/lib/Service/NotExportableException.php new file mode 100644 index 00000000..e4ccc7cb --- /dev/null +++ b/lib/Service/NotExportableException.php @@ -0,0 +1,32 @@ + + * + * @author Christopher Ng + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\UserMigration\Service; + +use Exception; + +class NotExportableException extends Exception { +} diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index cab9e7f8..a0022034 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -27,6 +27,8 @@ namespace OCA\UserMigration\Service; +use OC_Helper; +use OC_Util; use OC\AppFramework\Bootstrap\Coordinator; use OC\Cache\CappedMemoryCache; use OCA\UserMigration\BackgroundJob\UserExportJob; @@ -104,9 +106,10 @@ public function __construct( /** * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. + * * @return int Estimated size in KiB */ - public function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { + private function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { $cacheKey = $user->getUID(); if ($filteredMigratorList !== null) { $cacheKey .= '::' . json_encode($filteredMigratorList); @@ -132,6 +135,35 @@ public function estimateExportSize(IUser $user, ?array $filteredMigratorList = n /** * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. + * + * @throws NotExportableException + */ + public function checkExportability(IUser $user, ?array $filteredMigratorList = null): void { + try { + OC_Util::tearDownFS(); + OC_Util::setupFS($user->getUID()); + $storageInfo = OC_Helper::getStorageInfo('/'); + // TODO safe to cache? + $freeSpace = (int)ceil($storageInfo['free'] / 1024); + } catch (Throwable $e) { + throw new NotExportableException('Error calculating amount of free space available'); + } + + try { + $exportSize = $this->estimateExportSize($user, $filteredMigratorList); + } catch (UserMigrationException $e) { + throw new NotExportableException('Error estimating export size'); + } + + $freeSpaceAfterExport = $freeSpace - $exportSize; + if ($freeSpaceAfterExport < 0) { + throw new NotExportableException('Insufficient storage space available to export, please free up ' . (int)abs($freeSpaceAfterExport) . ' KiB or more to be able to export your data'); + } + } + + /** + * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. + * * @throws UserMigrationException */ public function export(IExportDestination $exportDestination, IUser $user, ?array $filteredMigratorList = null, ?OutputInterface $output = null): void { From 8253ee947470bc3fff7c700e5d9e39bbef398259 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Sat, 30 Apr 2022 00:32:24 +0000 Subject: [PATCH 11/22] Hoist not exportable exception Signed-off-by: Christopher Ng --- lib/Controller/ApiController.php | 2 +- lib/{Service => }/NotExportableException.php | 2 +- lib/Service/UserMigrationService.php | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) rename lib/{Service => }/NotExportableException.php (96%) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 6c9d7579..58319769 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -29,7 +29,7 @@ use OCA\UserMigration\AppInfo\Application; use OCA\UserMigration\Db\UserExport; use OCA\UserMigration\Db\UserImport; -use OCA\UserMigration\Service\NotExportableException; +use OCA\UserMigration\NotExportableException; use OCA\UserMigration\Service\UserMigrationService; use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataResponse; diff --git a/lib/Service/NotExportableException.php b/lib/NotExportableException.php similarity index 96% rename from lib/Service/NotExportableException.php rename to lib/NotExportableException.php index e4ccc7cb..a0afba4c 100644 --- a/lib/Service/NotExportableException.php +++ b/lib/NotExportableException.php @@ -24,7 +24,7 @@ * */ -namespace OCA\UserMigration\Service; +namespace OCA\UserMigration; use Exception; diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index a0022034..85a71f95 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -37,6 +37,7 @@ use OCA\UserMigration\Db\UserExportMapper; use OCA\UserMigration\Db\UserImport; use OCA\UserMigration\Db\UserImportMapper; +use OCA\UserMigration\NotExportableException; use OCP\AppFramework\Db\DoesNotExistException; use OCP\BackgroundJob\IJobList; use OCP\Files\IRootFolder; From 314168c7723033c45462f645f477753b3f77a947 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Thu, 2 Jun 2022 02:03:03 +0000 Subject: [PATCH 12/22] Ensure that migrator is an ISizeEstimationMigrator Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index 85a71f95..f1fe905d 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -48,6 +48,7 @@ use OCP\UserMigration\IExportDestination; use OCP\UserMigration\IImportSource; use OCP\UserMigration\IMigrator; +use OCP\UserMigration\ISizeEstimationMigrator; use OCP\UserMigration\TMigratorBasicVersionHandling; use OCP\UserMigration\UserMigrationException; use Psr\Container\ContainerInterface; @@ -127,7 +128,9 @@ private function estimateExportSize(IUser $user, ?array $filteredMigratorList = if ($filteredMigratorList !== null && !in_array($migrator->getId(), $filteredMigratorList)) { continue; } - $size += $migrator->getEstimatedExportSize($user); + if ($migrator instanceof ISizeEstimationMigrator) { + $size += $migrator->getEstimatedExportSize($user); + } } $this->internalCache->set($cacheKey, $size); From 28e1caea8c6bff33327cde53789498c0ac2d652f Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Thu, 2 Jun 2022 02:06:30 +0000 Subject: [PATCH 13/22] Use method to get free space Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index f1fe905d..cdd6dcf8 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -144,11 +144,8 @@ private function estimateExportSize(IUser $user, ?array $filteredMigratorList = */ public function checkExportability(IUser $user, ?array $filteredMigratorList = null): void { try { - OC_Util::tearDownFS(); - OC_Util::setupFS($user->getUID()); - $storageInfo = OC_Helper::getStorageInfo('/'); - // TODO safe to cache? - $freeSpace = (int)ceil($storageInfo['free'] / 1024); + $userFolder = $this->root->getUserFolder($user->getUID()); + $freeSpace = (int)ceil($userFolder->getFreeSpace() / 1024); } catch (Throwable $e) { throw new NotExportableException('Error calculating amount of free space available'); } From 14b7f2ac663d3aee00a57efeb02b9b53c44fd9ad Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Thu, 2 Jun 2022 02:25:17 +0000 Subject: [PATCH 14/22] Cache migrators separately Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index cdd6dcf8..f6fb81b9 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -112,15 +112,6 @@ public function __construct( * @return int Estimated size in KiB */ private function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { - $cacheKey = $user->getUID(); - if ($filteredMigratorList !== null) { - $cacheKey .= '::' . json_encode($filteredMigratorList); - } - - if ($this->internalCache->hasKey($cacheKey)) { - return $this->internalCache->get($cacheKey); - } - // 1MiB for base user data $size = 1024; @@ -128,12 +119,18 @@ private function estimateExportSize(IUser $user, ?array $filteredMigratorList = if ($filteredMigratorList !== null && !in_array($migrator->getId(), $filteredMigratorList)) { continue; } + $cacheKey = $user->getUID() . '::' . $migrator->getId(); + if ($this->internalCache->hasKey($cacheKey)) { + $size += $this->internalCache->get($cacheKey); + continue; + } if ($migrator instanceof ISizeEstimationMigrator) { - $size += $migrator->getEstimatedExportSize($user); + $migratorSize = $migrator->getEstimatedExportSize($user); + $this->internalCache->set($cacheKey, $migratorSize); + $size += $migratorSize; } } - $this->internalCache->set($cacheKey, $size); return $size; } From bb1a0788617a381ea3e1b5bf2d47f1367747dc61 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Thu, 2 Jun 2022 03:01:20 +0000 Subject: [PATCH 15/22] Add export estimate endpoint Signed-off-by: Christopher Ng --- appinfo/routes.php | 3 ++- lib/Controller/ApiController.php | 22 ++++++++++++++++++++++ lib/Service/UserMigrationService.php | 10 ++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 6c82e422..511f6d5c 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -33,7 +33,8 @@ ['name' => 'Api#migrators', 'url' => '/api/v{apiVersion}/migrators', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#status', 'url' => '/api/v{apiVersion}/status', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#cancel', 'url' => '/api/v{apiVersion}/cancel', 'verb' => 'PUT', 'requirements' => $requirements], - ['name' => 'Api#exportable', 'url' => '/api/v{apiVersion}/exportable', 'verb' => 'GET', 'requirements' => $requirements], + ['name' => 'Api#estimate', 'url' => '/api/v{apiVersion}/export/estimate', 'verb' => 'GET', 'requirements' => $requirements], + ['name' => 'Api#exportable', 'url' => '/api/v{apiVersion}/export', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#export', 'url' => '/api/v{apiVersion}/export', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'Api#import', 'url' => '/api/v{apiVersion}/import', 'verb' => 'POST', 'requirements' => $requirements], ], diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 58319769..647b7c9d 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -211,6 +211,28 @@ private function checkJobAndGetUser(): IUser { return $user; } + /** + * @NoAdminRequired + * @NoSubAdminRequired + * + * @throws OCSException + */ + public function estimate(?array $migrators): DataResponse { + $user = $this->checkJobAndGetUser(); + + if (!is_null($migrators)) { + $this->checkMigrators($migrators); + } + + try { + $size = $this->migrationService->estimateExportSize($user, $migrators); + } catch (UserMigrationException $e) { + throw new OCSException($e->getMessage()); + } + + return new DataResponse(['size' => $size], Http::STATUS_OK); + } + /** * @NoAdminRequired * @NoSubAdminRequired diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index f6fb81b9..c4055e51 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -110,8 +110,10 @@ public function __construct( * @param ?string[] $filteredMigratorList If not null, only these migrators will run. If empty only the main account data will be exported. * * @return int Estimated size in KiB + * + * @throws UserMigrationException */ - private function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { + public function estimateExportSize(IUser $user, ?array $filteredMigratorList = null): int { // 1MiB for base user data $size = 1024; @@ -125,7 +127,11 @@ private function estimateExportSize(IUser $user, ?array $filteredMigratorList = continue; } if ($migrator instanceof ISizeEstimationMigrator) { - $migratorSize = $migrator->getEstimatedExportSize($user); + try { + $migratorSize = $migrator->getEstimatedExportSize($user); + } catch (Throwable $e) { + throw new UserMigrationException('Could not estimate export size for ' . $migrator->getDisplayName(), 0, $e); + } $this->internalCache->set($cacheKey, $migratorSize); $size += $migratorSize; } From e427f6122029036e3b9ee0726fd3d8ffceb616df Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 3 Jun 2022 00:21:18 +0000 Subject: [PATCH 16/22] Consolidate estimation and exportability warnings into a single endpoint Avoids having to query multiple endpoints on the client for closely related information Signed-off-by: Christopher Ng --- appinfo/routes.php | 1 - lib/Controller/ApiController.php | 25 ++++++------------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 511f6d5c..19751966 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -33,7 +33,6 @@ ['name' => 'Api#migrators', 'url' => '/api/v{apiVersion}/migrators', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#status', 'url' => '/api/v{apiVersion}/status', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#cancel', 'url' => '/api/v{apiVersion}/cancel', 'verb' => 'PUT', 'requirements' => $requirements], - ['name' => 'Api#estimate', 'url' => '/api/v{apiVersion}/export/estimate', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#exportable', 'url' => '/api/v{apiVersion}/export', 'verb' => 'GET', 'requirements' => $requirements], ['name' => 'Api#export', 'url' => '/api/v{apiVersion}/export', 'verb' => 'POST', 'requirements' => $requirements], ['name' => 'Api#import', 'url' => '/api/v{apiVersion}/import', 'verb' => 'POST', 'requirements' => $requirements], diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 647b7c9d..4a7b8e36 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -217,7 +217,7 @@ private function checkJobAndGetUser(): IUser { * * @throws OCSException */ - public function estimate(?array $migrators): DataResponse { + public function exportable(?array $migrators): DataResponse { $user = $this->checkJobAndGetUser(); if (!is_null($migrators)) { @@ -230,29 +230,16 @@ public function estimate(?array $migrators): DataResponse { throw new OCSException($e->getMessage()); } - return new DataResponse(['size' => $size], Http::STATUS_OK); - } - - /** - * @NoAdminRequired - * @NoSubAdminRequired - * - * @throws OCSException - */ - public function exportable(?array $migrators): DataResponse { - $user = $this->checkJobAndGetUser(); - - if (!is_null($migrators)) { - $this->checkMigrators($migrators); - } - try { $this->migrationService->checkExportability($user, $migrators); } catch (NotExportableException $e) { - throw new OCSException($e->getMessage()); + $warning = $e->getMessage(); } - return new DataResponse([], Http::STATUS_OK); + return new DataResponse([ + 'size' => $size, + 'warning' => $warning ?? null, + ], Http::STATUS_OK); } /** From 3858dd90514920ce72090ac8aa0def6670ced8b0 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 3 Jun 2022 00:42:11 +0000 Subject: [PATCH 17/22] Round estimated size before sending response and add units Signed-off-by: Christopher Ng --- lib/Controller/ApiController.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 4a7b8e36..fb2dfd30 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -237,7 +237,8 @@ public function exportable(?array $migrators): DataResponse { } return new DataResponse([ - 'size' => $size, + 'size' => round($size, -2), + 'units' => 'KiB', 'warning' => $warning ?? null, ], Http::STATUS_OK); } From b71b032b9c0aba234be46e27b50351822200d9e1 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 3 Jun 2022 01:02:53 +0000 Subject: [PATCH 18/22] Implement ISizeEstimationMigrator and fix files versions estimation Signed-off-by: Christopher Ng --- lib/Migrator/FilesMigrator.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Migrator/FilesMigrator.php b/lib/Migrator/FilesMigrator.php index c831847d..f1e668e7 100644 --- a/lib/Migrator/FilesMigrator.php +++ b/lib/Migrator/FilesMigrator.php @@ -45,11 +45,12 @@ use OCP\UserMigration\IExportDestination; use OCP\UserMigration\IImportSource; use OCP\UserMigration\IMigrator; +use OCP\UserMigration\ISizeEstimationMigrator; use OCP\UserMigration\TMigratorBasicVersionHandling; use OCP\UserMigration\UserMigrationException; use Symfony\Component\Console\Output\OutputInterface; -class FilesMigrator implements IMigrator { +class FilesMigrator implements IMigrator, ISizeEstimationMigrator { use TMigratorBasicVersionHandling; protected const PATH_FILES = Application::APP_ID.'/files'; @@ -111,11 +112,10 @@ public function getEstimatedExportSize(IUser $user): int { try { $versionsFolder = $this->root->get('/'.$uid.'/'.FilesVersionsStorage::VERSIONS_ROOT); if ($versionsFolder instanceof Folder) { - return 0; + $size += $versionsFolder->getSize() / 1024; } - $size += $versionsFolder->getSize() / 1024; } catch (\Throwable $e) { - return 0; + // Skip versions folder size estimate on failure } // 1MiB for tags and system tags From 96a5608c1ef17ff2d9c73f17a17882f7978defdc Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Fri, 3 Jun 2022 01:10:00 +0000 Subject: [PATCH 19/22] Remove unused use statements Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index c4055e51..147f2039 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -27,8 +27,6 @@ namespace OCA\UserMigration\Service; -use OC_Helper; -use OC_Util; use OC\AppFramework\Bootstrap\Coordinator; use OC\Cache\CappedMemoryCache; use OCA\UserMigration\BackgroundJob\UserExportJob; From b61905a521b8f4237c908f289240d683b62ef628 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Mon, 6 Jun 2022 23:59:25 +0000 Subject: [PATCH 20/22] Convert KiB to MiB Signed-off-by: Christopher Ng --- lib/Controller/ApiController.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index fb2dfd30..6208665a 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -226,6 +226,8 @@ public function exportable(?array $migrators): DataResponse { try { $size = $this->migrationService->estimateExportSize($user, $migrators); + // Convert to MiB and round to one significant digit after the decimal point + $roundedSize = round($size / 1024, 1); } catch (UserMigrationException $e) { throw new OCSException($e->getMessage()); } @@ -237,8 +239,8 @@ public function exportable(?array $migrators): DataResponse { } return new DataResponse([ - 'size' => round($size, -2), - 'units' => 'KiB', + 'size' => $roundedSize, + 'units' => 'MiB', 'warning' => $warning ?? null, ], Http::STATUS_OK); } From 02478f85fbebe646c155142bc0968eb1a512355d Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Wed, 8 Jun 2022 01:11:15 +0000 Subject: [PATCH 21/22] Explicitly name estimated size Signed-off-by: Christopher Ng --- lib/Controller/ApiController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Controller/ApiController.php b/lib/Controller/ApiController.php index 6208665a..766e485f 100644 --- a/lib/Controller/ApiController.php +++ b/lib/Controller/ApiController.php @@ -239,7 +239,7 @@ public function exportable(?array $migrators): DataResponse { } return new DataResponse([ - 'size' => $roundedSize, + 'estimatedSize' => $roundedSize, 'units' => 'MiB', 'warning' => $warning ?? null, ], Http::STATUS_OK); From 85c551bd55e379fdbd54f02046d73bae65409a51 Mon Sep 17 00:00:00 2001 From: Christopher Ng Date: Wed, 8 Jun 2022 01:12:30 +0000 Subject: [PATCH 22/22] Add previous export size to free space Signed-off-by: Christopher Ng --- lib/Service/UserMigrationService.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/Service/UserMigrationService.php b/lib/Service/UserMigrationService.php index 147f2039..e984fa04 100644 --- a/lib/Service/UserMigrationService.php +++ b/lib/Service/UserMigrationService.php @@ -35,10 +35,13 @@ use OCA\UserMigration\Db\UserExportMapper; use OCA\UserMigration\Db\UserImport; use OCA\UserMigration\Db\UserImportMapper; +use OCA\UserMigration\ExportDestination; use OCA\UserMigration\NotExportableException; use OCP\AppFramework\Db\DoesNotExistException; use OCP\BackgroundJob\IJobList; +use OCP\Files\File; use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IConfig; use OCP\IUser; use OCP\IUserManager; @@ -151,6 +154,17 @@ public function checkExportability(IUser $user, ?array $filteredMigratorList = n throw new NotExportableException('Error calculating amount of free space available'); } + try { + $exportFile = $userFolder->get(ExportDestination::EXPORT_FILENAME); + if (!($exportFile instanceof File)) { + throw new \InvalidArgumentException('User export is not a file'); + } + // Add previous export file size to free space as it will be overwritten if existing + $freeSpace += $exportFile->getSize() / 1024; + } catch (NotFoundException $e) { + // No size addition needed if export file doesn't exist + } + try { $exportSize = $this->estimateExportSize($user, $filteredMigratorList); } catch (UserMigrationException $e) {