diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 0e08c6a8cfe41..23604a82df9b5 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -12,6 +12,7 @@ use OC\DB\ConnectionAdapter; use OC\Files\Utils\Scanner; use OC\ForbiddenException; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -32,10 +33,12 @@ class ScanAppData extends Base { protected int $foldersCounter = 0; protected int $filesCounter = 0; + protected int $previewsCounter = -1; public function __construct( protected IRootFolder $rootFolder, protected IConfig $config, + private StorageFactory $previewStorage, ) { parent::__construct(); } @@ -51,6 +54,14 @@ protected function configure(): void { } protected function scanFiles(OutputInterface $output, string $folder): int { + if ($folder === 'preview' || $folder === '') { + $this->previewsCounter = $this->previewStorage->scan(); + + if ($folder === 'preview') { + return self::SUCCESS; + } + } + try { /** @var Folder $appData */ $appData = $this->getAppDataFolder(); @@ -134,7 +145,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->initTools(); $exitCode = $this->scanFiles($output, $folder); - if ($exitCode === 0) { + if ($exitCode === self::SUCCESS) { $this->presentStats($output); } return $exitCode; @@ -162,7 +173,7 @@ protected function initTools(): void { * * @throws \ErrorException */ - public function exceptionErrorHandler($severity, $message, $file, $line) { + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; @@ -173,10 +184,12 @@ public function exceptionErrorHandler($severity, $message, $file, $line) { protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); - - $headers = [ - 'Folders', 'Files', 'Elapsed time' - ]; + if ($this->previewsCounter !== -1) { + $headers[] = 'Previews'; + } + $headers[] = 'Folders'; + $headers[] = 'Files'; + $headers[] = 'Elapsed time'; $this->showSummary($headers, null, $output); } @@ -187,14 +200,15 @@ protected function presentStats(OutputInterface $output): void { * @param string[] $headers * @param string[] $rows */ - protected function showSummary($headers, $rows, OutputInterface $output): void { + protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void { $niceDate = $this->formatExecTime(); if (!$rows) { - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $niceDate, - ]; + if ($this->previewsCounter !== -1) { + $rows[] = $this->previewsCounter; + } + $rows[] = $this->foldersCounter; + $rows[] = $this->filesCounter; + $rows[] = $niceDate; } $table = new Table($output); $table diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 621e368f0a3f5..8be3f1e9c1d13 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1240,9 +1240,11 @@ + + + - @@ -2708,17 +2710,6 @@ timeFactory->getTime()]]> - - - - - - - - - - - @@ -3721,16 +3712,6 @@ - - - - - - - - - - @@ -3991,14 +3972,6 @@ - - - mode]]> - - - - - diff --git a/build/stubs/php-polyfill.php b/build/stubs/php-polyfill.php new file mode 100644 index 0000000000000..606a21f8dfeb3 --- /dev/null +++ b/build/stubs/php-polyfill.php @@ -0,0 +1,9 @@ +appData = $appDataFactory->get('preview'); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + $this->setInterval(24 * 60 * 60); + $this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/'; + } + + #[Override] + protected function run(mixed $argument): void { + if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { + return; + } + + $startTime = time(); + while (true) { + $qb = $this->connection->getQueryBuilder(); + $qb->select('path') + ->from('filecache') + // Hierarchical preview folder structure + ->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%'))) + // Legacy flat preview folder structure + ->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%'))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->setMaxResults(100); + + $result = $qb->executeQuery(); + $foundPreviews = $this->processQueryResult($result); + + if (!$foundPreviews) { + break; + } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 3600) { + return; + } + } + + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + } + + private function processQueryResult(IResult $result): bool { + $foundPreview = false; + $fileIds = []; + $flatFileIds = []; + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = (int)$pathSplit[count($pathSplit) - 2]; + if (count($pathSplit) === 11) { + // Hierarchical structure + if (!in_array($fileId, $fileIds)) { + $fileIds[] = $fileId; + } + } else { + // Flat structure + if (!in_array($fileId, $flatFileIds)) { + $flatFileIds[] = $fileId; + } + } + $foundPreview = true; + } + + foreach ($fileIds as $fileId) { + $this->processPreviews($fileId, flatPath: false); + } + + foreach ($flatFileIds as $fileId) { + $this->processPreviews($fileId, flatPath: true); + } + return $foundPreview; + } + + /** + * @param array $previewFolders + */ + private function processPreviews(int $fileId, bool $flatPath): void { + $internalPath = $this->getInternalFolder((string)$fileId, $flatPath); + $folder = $this->appData->getFolder($internalPath); + + /** + * @var list $previewFiles + */ + $previewFiles = []; + + foreach ($folder->getDirectoryListing() as $previewFile) { + $path = $fileId . '/' . $previewFile->getName(); + /** @var SimpleFile $previewFile */ + $preview = Preview::fromPath($path, $this->mimeTypeDetector); + if (!$preview) { + $this->logger->error('Unable to import old preview at path.'); + continue; + } + $preview->setSize($previewFile->getSize()); + $preview->setMtime($previewFile->getMtime()); + $preview->setOldFileId($previewFile->getId()); + $preview->setEncrypted(false); + + $previewFiles[] = [ + 'file' => $previewFile, + 'preview' => $preview, + ]; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->select('storage', 'etag', 'mimetype') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->setMaxResults(1); + + $result = $qb->executeQuery(); + $result = $result->fetchAll(); + + if (count($result) > 0) { + foreach ($previewFiles as $previewFile) { + /** @var Preview $preview */ + $preview = $previewFile['preview']; + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result[0]['mimetype'])); + try { + $preview = $this->previewMapper->insert($preview); + } catch (Exception) { + // We already have this preview in the preview table, skip + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); + continue; + } + + try { + $this->storageFactory->migratePreview($preview, $file); + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); + // Do not call $file->delete() as this will also delete the file from the file system + } catch (\Exception $e) { + $this->previewMapper->delete($preview); + throw $e; + } + } + } else { + // No matching fileId, delete preview + try { + $this->connection->beginTransaction(); + foreach ($previewFiles as $previewFile) { + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $file->delete(); + } + $this->connection->commit(); + } catch (Exception) { + $this->connection->rollback(); + } + } + + $this->deleteFolder($internalPath); + } + + public static function getInternalFolder(string $name, bool $flatPath): string { + if ($flatPath) { + return $name; + } + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + private function deleteFolder(string $path): void { + $current = $path; + + while (true) { + $appDataPath = $this->previewRootPath . $current; + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($appDataPath)))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); + + $current = dirname($current); + if ($current === '/' || $current === '.' || $current === '') { + break; + } + + $folder = $this->appData->getFolder($current); + if (count($folder->getDirectoryListing()) !== 0) { + break; + } + } + } +} diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php deleted file mode 100644 index a92a4cf8ed0c2..0000000000000 --- a/core/Command/Preview/Repair.php +++ /dev/null @@ -1,293 +0,0 @@ -memoryLimit = (int)$phpIni->getBytes('memory_limit'); - $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024; - - parent::__construct(); - } - - protected function configure() { - $this - ->setName('preview:repair') - ->setDescription('distributes the existing previews into subfolders') - ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.') - ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.') - ->addOption('delete', null, InputOption::VALUE_NONE, 'Delete instead of migrating them. Usefull if too many entries to migrate.'); - } - - protected function execute(InputInterface $input, OutputInterface $output): int { - if ($this->memoryLimit !== -1) { - $limitInMiB = round($this->memoryLimit / 1024 / 1024, 1); - $thresholdInMiB = round($this->memoryTreshold / 1024 / 1024, 1); - $output->writeln("Memory limit is $limitInMiB MiB"); - $output->writeln("Memory threshold is $thresholdInMiB MiB"); - $output->writeln(''); - $memoryCheckEnabled = true; - } else { - $output->writeln('No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.'); - $output->writeln(''); - $memoryCheckEnabled = false; - } - - $dryMode = $input->getOption('dry'); - $deleteMode = $input->getOption('delete'); - - - if ($dryMode) { - $output->writeln('INFO: The migration is run in dry mode and will not modify anything.'); - $output->writeln(''); - } elseif ($deleteMode) { - $output->writeln('WARN: The migration will _DELETE_ old previews.'); - $output->writeln(''); - } - - $instanceId = $this->config->getSystemValueString('instanceid'); - - $output->writeln('This will migrate all previews from the old preview location to the new one.'); - $output->writeln(''); - - $output->writeln('Fetching previews that need to be migrated …'); - /** @var Folder $currentPreviewFolder */ - $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview"); - - $directoryListing = $currentPreviewFolder->getDirectoryListing(); - - $total = count($directoryListing); - /** - * by default there could be 0-9 a-f and the old-multibucket folder which are all fine - */ - if ($total < 18) { - $directoryListing = array_filter($directoryListing, function ($dir) { - if ($dir->getName() === 'old-multibucket') { - return false; - } - - // a-f can't be a file ID -> removing from migration - if (preg_match('!^[a-f]$!', $dir->getName())) { - return false; - } - - if (preg_match('!^[0-9]$!', $dir->getName())) { - // ignore folders that only has folders in them - if ($dir instanceof Folder) { - foreach ($dir->getDirectoryListing() as $entry) { - if (!$entry instanceof Folder) { - return true; - } - } - return false; - } - } - return true; - }); - $total = count($directoryListing); - } - - if ($total === 0) { - $output->writeln('All previews are already migrated.'); - return 0; - } - - $output->writeln("A total of $total preview files need to be migrated."); - $output->writeln(''); - $output->writeln('The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This will finish the current batch and then stop the migration. This migration can then just be started and it will continue.'); - - if ($input->getOption('batch')) { - $output->writeln('Batch mode active: migration is started right away.'); - } else { - /** @var QuestionHelper $helper */ - $helper = $this->getHelper('question'); - $question = new ConfirmationQuestion('Should the migration be started? (y/[n]) ', false); - - if (!$helper->ask($input, $output, $question)) { - return 0; - } - } - - // register the SIGINT listener late in here to be able to exit in the early process of this command - pcntl_signal(SIGINT, [$this, 'sigIntHandler']); - - $output->writeln(''); - $output->writeln(''); - $section1 = $output->section(); - $section2 = $output->section(); - $progressBar = new ProgressBar($section2, $total); - $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%'); - $time = (new \DateTime())->format('H:i:s'); - $progressBar->setMessage("$time Starting …"); - $progressBar->maxSecondsBetweenRedraws(0.2); - $progressBar->start(); - - foreach ($directoryListing as $oldPreviewFolder) { - pcntl_signal_dispatch(); - $name = $oldPreviewFolder->getName(); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Migrating previews of file with fileId $name …"); - $progressBar->display(); - - if ($this->stopSignalReceived) { - $section1->writeln("$time Stopping migration …"); - return 0; - } - if (!$oldPreviewFolder instanceof Folder) { - $section1->writeln(" Skipping non-folder $name …"); - $progressBar->advance(); - continue; - } - if ($name === 'old-multibucket') { - $section1->writeln(" Skipping fallback mount point $name …"); - $progressBar->advance(); - continue; - } - if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) { - $section1->writeln(" Skipping hex-digit folder $name …"); - $progressBar->advance(); - continue; - } - if (!preg_match('!^\d+$!', $name)) { - $section1->writeln(" Skipping non-numeric folder $name …"); - $progressBar->advance(); - continue; - } - - $newFoldername = Root::getInternalFolder($name); - - $memoryUsage = memory_get_usage(); - if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) { - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(' Stopped process 25 MB before reaching the memory limit to avoid a hard crash.'); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Reached memory limit and stopped to avoid hard crash."); - return 1; - } - - $lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId(); - try { - $section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE); - $this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - } catch (LockedException $e) { - $section1->writeln(' Skipping because it is locked - another process seems to work on this …'); - continue; - } - - $previews = $oldPreviewFolder->getDirectoryListing(); - if ($previews !== []) { - try { - $this->rootFolder->get("appdata_$instanceId/preview/$newFoldername"); - } catch (NotFoundException $e) { - $section1->writeln(" Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - $this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername"); - } - } - - foreach ($previews as $preview) { - pcntl_signal_dispatch(); - $previewName = $preview->getName(); - - if ($preview instanceof Folder) { - $section1->writeln(" Skipping folder $name/$previewName …"); - $progressBar->advance(); - continue; - } - - // Execute process - if (!$dryMode) { - // Delete preview instead of moving - if ($deleteMode) { - try { - $section1->writeln(" Delete preview/$name/$previewName", OutputInterface::VERBOSITY_VERBOSE); - $preview->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete preview at preview/$name/$previewName", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } else { - try { - $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); - } catch (\Exception $e) { - $this->logger->error("Failed to move preview from preview/$name/$previewName to preview/$newFoldername", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - } - } - - if ($oldPreviewFolder->getDirectoryListing() === []) { - $section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - try { - $oldPreviewFolder->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete empty folder preview/$name", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - - $this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - $section1->writeln(' Unlocked', OutputInterface::VERBOSITY_VERBOSE); - - $section1->writeln(" Finished migrating previews of file with fileId $name …"); - $progressBar->advance(); - } - - $progressBar->finish(); - $output->writeln(''); - return 0; - } - - protected function sigIntHandler() { - echo "\nSignal received - will finish the step and then stop the migration.\n\n\n"; - $this->stopSignalReceived = true; - } -} diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index ce874e15406f9..978128a609fd6 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -8,14 +8,14 @@ */ namespace OC\Core\Command\Preview; -use OC\Preview\Storage\Root; -use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\Files\IMimeTypeLoader; +use OC\Preview\Db\Preview; +use OC\Preview\PreviewService; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IAvatarManager; use OCP\IDBConnection; use OCP\IUserManager; +use Override; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -23,22 +23,23 @@ class ResetRenderedTexts extends Command { public function __construct( - protected IDBConnection $connection, - protected IUserManager $userManager, - protected IAvatarManager $avatarManager, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, + protected readonly IDBConnection $connection, + protected readonly IUserManager $userManager, + protected readonly IAvatarManager $avatarManager, + private readonly PreviewService $previewService, ) { parent::__construct(); } - protected function configure() { + #[Override] + protected function configure(): void { $this ->setName('preview:reset-rendered-texts') ->setDescription('Deletes all generated avatars and previews of text and md files') ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.'); } + #[Override] protected function execute(InputInterface $input, OutputInterface $output): int { $dryMode = $input->getOption('dry'); @@ -67,9 +68,7 @@ private function deleteAvatars(OutputInterface $output, bool $dryMode): void { try { $avatar->remove(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { + } catch (NotFoundException|NotPermittedException) { // continue } } @@ -91,8 +90,8 @@ private function getAvatarsToDelete(): \Iterator { private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; - foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) { - $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); + foreach ($this->getPreviewsToDelete() as $preview) { + $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -100,65 +99,20 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { continue; } - try { - $preview = $this->previewFolder->getFolder((string)$previewFileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue - } + $this->previewService->deletePreview($preview); } $output->writeln('Deleted ' . $previewsToDeleteCount . ' previews'); } - // Copy pasted and adjusted from - // "lib/private/Preview/BackgroundCleanupJob.php". - private function getPreviewsToDelete(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') - ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name', 'b.path') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) - ->where( - $qb->expr()->andX( - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->orX( - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) - ) - ) - ); - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - yield $row; - } - - $cursor->closeCursor(); + /** + * @return \Generator + */ + private function getPreviewsToDelete(): \Generator { + return $this->previewService->getPreviewsForMimeTypes([ + 'text/plain', + 'text/markdown', + 'text/x-markdown' + ]); } } diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php new file mode 100644 index 0000000000000..32a25b0e33c47 --- /dev/null +++ b/core/Migrations/Version33000Date20250819110529.php @@ -0,0 +1,73 @@ +hasTable('preview_locations')) { + $table = $schema->createTable('preview_locations'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('bucket_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('object_store_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->setPrimaryKey(['id']); + } + + if (!$schema->hasTable('preview_versions')) { + $table = $schema->createTable('preview_versions'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('version', Types::STRING, ['notnull' => true, 'default' => '', 'length' => 1024]); + $table->setPrimaryKey(['id']); + } + + if (!$schema->hasTable('previews')) { + $table = $schema->createTable('previews'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); + $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); + $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('mimetype_id', Types::INTEGER, ['notnull' => true]); + $table->addColumn('source_mimetype_id', Types::INTEGER, ['notnull' => true]); + $table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]); + $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('version_id', Types::BIGINT, ['notnull' => true, 'default' => -1]); + $table->setPrimaryKey(['id']); + $table->addIndex(['file_id']); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype_id', 'cropped', 'version_id'], 'previews_file_uniq_idx'); + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index 9fd5b9b611e0e..f6c0b9466b5cb 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -206,7 +206,6 @@ $application->add(Server::get(Command\Preview\Cleanup::class)); $application->add(Server::get(Generate::class)); - $application->add(Server::get(Command\Preview\Repair::class)); $application->add(Server::get(ResetRenderedTexts::class)); $application->add(Server::get(Add::class)); diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index ee77fbd4cda82..4c2e473240bd7 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1253,6 +1253,7 @@ 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => $baseDir . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1343,7 +1344,6 @@ 'OC\\Core\\Command\\Memcache\\RedisCommand' => $baseDir . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => $baseDir . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => $baseDir . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => $baseDir . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => $baseDir . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => $baseDir . '/core/Command/Router/MatchRoute.php', @@ -1528,6 +1528,7 @@ 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1685,7 +1686,6 @@ 'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => $baseDir . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php', @@ -1880,6 +1880,8 @@ 'OC\\Preview\\BackgroundCleanupJob' => $baseDir . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => $baseDir . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => $baseDir . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => $baseDir . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => $baseDir . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => $baseDir . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => $baseDir . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => $baseDir . '/lib/private/Preview/GIF.php', @@ -1906,13 +1908,18 @@ 'OC\\Preview\\PNG' => $baseDir . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => $baseDir . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => $baseDir . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => $baseDir . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => $baseDir . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => $baseDir . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', - 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', + 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', @@ -1950,6 +1957,7 @@ 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => $baseDir . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 3b18f00da9697..2cb14099c01e2 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1294,6 +1294,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1384,7 +1385,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Memcache\\RedisCommand' => __DIR__ . '/../../..' . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => __DIR__ . '/../../..' . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => __DIR__ . '/../../..' . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => __DIR__ . '/../../..' . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => __DIR__ . '/../../..' . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => __DIR__ . '/../../..' . '/core/Command/Router/MatchRoute.php', @@ -1569,6 +1569,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1726,7 +1727,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php', @@ -1921,6 +1921,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\BackgroundCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => __DIR__ . '/../../..' . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => __DIR__ . '/../../..' . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => __DIR__ . '/../../..' . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => __DIR__ . '/../../..' . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => __DIR__ . '/../../..' . '/lib/private/Preview/GIF.php', @@ -1947,13 +1949,18 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\PNG' => __DIR__ . '/../../..' . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => __DIR__ . '/../../..' . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => __DIR__ . '/../../..' . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => __DIR__ . '/../../..' . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => __DIR__ . '/../../..' . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php', 'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', - 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', + 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', @@ -1991,6 +1998,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 3f4f70b865b01..d5f7d40e1b650 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -8,7 +8,20 @@ */ namespace OC\Files\Cache; +use OCP\IConfig; +use OCP\Server; +use Override; + class LocalRootScanner extends Scanner { + private string $previewFolder; + + public function __construct(\OC\Files\Storage\Storage $storage) { + parent::__construct($storage); + $config = Server::get(IConfig::class); + $this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview'; + } + + #[Override] public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { if ($this->shouldScanPath($file)) { return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); @@ -17,6 +30,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = } } + #[Override] public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { if ($this->shouldScanPath($path)) { return parent::scan($path, $recursive, $reuse, $lock); @@ -25,6 +39,14 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc } } + #[Override] + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + if (str_starts_with($path, $this->previewFolder)) { + return 0; + } + return parent::scanChildren($path, $recursive, $reuse, $folderId, $lock, $oldSize, $etagChanged); + } + private function shouldScanPath(string $path): bool { $path = trim($path, '/'); return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index b067f70b8cb51..2fa0dd09e4fd0 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -11,14 +11,15 @@ use OC\Files\Storage\Wrapper\Encryption; use OC\Files\Storage\Wrapper\Jail; use OC\Hooks\BasicEmitter; -use OC\SystemConfig; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; use OCP\Files\NotFoundException; use OCP\Files\Storage\ILockingStorage; use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use OCP\Server; use Psr\Log\LoggerInterface; /** @@ -69,12 +70,11 @@ public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); - /** @var SystemConfig $config */ - $config = \OC::$server->get(SystemConfig::class); - $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); - $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); - $this->lockingProvider = \OC::$server->get(ILockingProvider::class); - $this->connection = \OC::$server->get(IDBConnection::class); + $config = Server::get(IConfig::class); + $this->cacheActive = !$config->getSystemValueBool('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getSystemValueBool('filescanner_no_transactions', false); + $this->lockingProvider = Server::get(ILockingProvider::class); + $this->connection = Server::get(IDBConnection::class); } /** @@ -318,7 +318,6 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc try { $data = $this->scanFile($path, $reuse, -1, lock: $lock); - if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php deleted file mode 100644 index 1546ef98f5020..0000000000000 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ /dev/null @@ -1,138 +0,0 @@ -logger = $logger; - $this->config = $config; - } - - /** - * @return MountPoint[] - * @throws \Exception - */ - public function getRootMounts(IStorageFactory $loader): array { - if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { - return []; - } - if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { - return []; - } - - $instanceId = $this->config->getSystemValueString('instanceid', ''); - $mountPoints = []; - $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; - $i = 0; - foreach ($directoryRange as $parent) { - foreach ($directoryRange as $child) { - $mountPoints[] = new MountPoint( - AppdataPreviewObjectStoreStorage::class, - '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, - $this->getMultiBucketObjectStore($i), - $loader, - null, - null, - self::class - ); - $i++; - } - } - - $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); - $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); - $fakeRootStorageJail = new Jail([ - 'storage' => $fakeRootStorage, - 'root' => '/appdata_' . $instanceId . '/preview', - ]); - - // add a fallback location to be able to fetch existing previews from the old bucket - $mountPoints[] = new MountPoint( - $fakeRootStorageJail, - '/appdata_' . $instanceId . '/preview/old-multibucket', - null, - $loader, - null, - null, - self::class - ); - - return $mountPoints; - } - - protected function getMultiBucketObjectStore(int $number): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - - $config['arguments']['bucket'] .= "-preview-$number"; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['internal-id'] = $number; - - return $config['arguments']; - } - - protected function getMultiBucketObjectStoreForRoot(): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - $config['arguments']['bucket'] .= '0'; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - return $config['arguments']; - } -} diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 008431b3fbf4f..02bc28f376e74 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix?: string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( @@ -119,12 +119,14 @@ public function getObjectStoreConfigs(): ?array { 'default' => 'server1', 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), 'root' => 'server1', + 'preview' => 'server1', ]; } elseif ($objectStore) { if (!isset($objectStore['default'])) { $objectStore = [ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => $objectStore, ]; } @@ -132,6 +134,10 @@ public function getObjectStoreConfigs(): ?array { $objectStore['root'] = 'default'; } + if (!isset($objectStore['preview'])) { + $objectStore['preview'] = 'default'; + } + if (!is_string($objectStore['default'])) { throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); } @@ -155,7 +161,7 @@ public function getObjectStoreConfigs(): ?array { } /** - * @param array|string $config + * @param array{multibucket?: bool, objectPrefix?: string, ...}|string $config * @return string|ObjectStoreConfig */ private function validateObjectStoreConfig(array|string $config): array|string { diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 3138abb1bf988..f6122fd0e1255 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -8,23 +8,23 @@ */ namespace OC\Preview; -use OC\Preview\Storage\Root; +use OC\Preview\Db\Preview; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\Files\IMimeTypeLoader; -use OCP\Files\NotFoundException; -use OCP\Files\NotPermittedException; use OCP\IDBConnection; +/** + * @psalm-type FileId int + * @psalm-type StorageId int + */ class BackgroundCleanupJob extends TimedJob { public function __construct( ITimeFactory $timeFactory, - private IDBConnection $connection, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, - private bool $isCLI, + readonly private IDBConnection $connection, + readonly private PreviewService $previewService, + readonly private bool $isCLI, ) { parent::__construct($timeFactory); // Run at most once an hour @@ -32,88 +32,29 @@ public function __construct( $this->setTimeSensitivity(self::TIME_INSENSITIVE); } - public function run($argument) { + public function run($argument): void { foreach ($this->getDeletedFiles() as $fileId) { - try { - $preview = $this->previewFolder->getFolder((string)$fileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue + $previewIds = []; + foreach ($this->previewService->getAvailablePreviewsForFile($fileId) as $preview) { + $this->previewService->deletePreview($preview); } } } + /** + * @return \Iterator + */ private function getDeletedFiles(): \Iterator { - yield from $this->getOldPreviewLocations(); - yield from $this->getNewPreviewLocations(); - } - - private function getOldPreviewLocations(): \Iterator { - if ($this->connection->getShardDefinition('filecache')) { - // sharding is new enough that we don't need to support this - return; - } - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) - ->where( - $qb->expr()->andX( - $qb->expr()->isNull('b.fileid'), - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())), - $qb->expr()->like('a.name', $qb->createNamedParameter('__%')), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); - - if (!$this->isCLI) { - $qb->setMaxResults(10); - } - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - yield $row['name']; - } - - $cursor->closeCursor(); - } - - private function getNewPreviewLocations(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') - ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - if ($this->connection->getShardDefinition('filecache')) { - $chunks = $this->getAllPreviewIds($data['path'], 1000); - foreach ($chunks as $chunk) { - yield from $this->findMissingSources($chunk); + foreach ($this->previewService->getAvailableFileIds() as $availableFileIdGroup) { + $fileIds = $this->findMissingSources($availableFileIdGroup['storageId'], $availableFileIdGroup['fileIds']); + foreach ($fileIds as $fileId) { + yield $fileId; + } } - return; } - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - /* * Deleting a file will not delete related previews right away. * @@ -130,71 +71,36 @@ private function getNewPreviewLocations(): \Iterator { * If the related file is deleted, b.fileid will be null and the preview folder can be deleted. */ $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + $qb->select('p.file_id') + ->from('previews', 'p') + ->leftJoin('p', 'filecache', 'f', $qb->expr()->eq( + 'p.file_id', 'f.fileid' )) - ->where( - $qb->expr()->andX( - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->isNull('b.fileid'), - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); + ->where($qb->expr()->isNull('f.fileid')); if (!$this->isCLI) { $qb->setMaxResults(10); } $cursor = $qb->executeQuery(); - while ($row = $cursor->fetch()) { - yield $row['name']; + yield (int)$row['file_id']; } - $cursor->closeCursor(); } - private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterator { - // See `getNewPreviewLocations` for some more info about the logic here - $like = $this->connection->escapeLikeParameter($previewRoot) . '/_/_/_/_/_/_/_/%'; - - $qb = $this->connection->getQueryBuilder(); - $qb->select('name', 'fileid') - ->from('filecache') - ->where( - $qb->expr()->andX( - $qb->expr()->eq('storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->like('path', $qb->createNamedParameter($like)), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->gt('fileid', $qb->createParameter('min_id')), - ) - ) - ->orderBy('fileid', 'ASC') - ->setMaxResults($chunkSize); - - $minId = 0; - while (true) { - $qb->setParameter('min_id', $minId); - $rows = $qb->executeQuery()->fetchAll(); - if (count($rows) > 0) { - $minId = $rows[count($rows) - 1]['fileid']; - yield array_map(function ($row) { - return (int)$row['name']; - }, $rows); - } else { - break; - } - } - } - - private function findMissingSources(array $ids): array { + /** + * @param FileId[] $ids + * @return FileId[] + */ + private function findMissingSources(int $storage, array $ids): array { $qb = $this->connection->getQueryBuilder(); $qb->select('fileid') ->from('filecache') - ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + ->where($qb->expr()->andX( + $qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)), + $qb->expr()->eq('storage', $qb->createNamedParameter($storage, IQueryBuilder::PARAM_INT)), + )); $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); return array_diff($ids, $found); } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php new file mode 100644 index 0000000000000..d3b3ab5ad07b2 --- /dev/null +++ b/lib/private/Preview/Db/Preview.php @@ -0,0 +1,179 @@ +addType('fileId', Types::BIGINT); + $this->addType('storageId', Types::BIGINT); + $this->addType('oldFileId', Types::BIGINT); + $this->addType('locationId', Types::BIGINT); + $this->addType('width', Types::INTEGER); + $this->addType('height', Types::INTEGER); + $this->addType('mimetypeId', Types::INTEGER); + $this->addType('sourceMimetypeId', Types::INTEGER); + $this->addType('mtime', Types::INTEGER); + $this->addType('size', Types::INTEGER); + $this->addType('max', Types::BOOLEAN); + $this->addType('cropped', Types::BOOLEAN); + $this->addType('encrypted', Types::BOOLEAN); + $this->addType('etag', Types::STRING); + $this->addType('versionId', Types::STRING); + } + + public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector): Preview|false { + $preview = new self(); + $preview->setFileId((int)basename(dirname($path))); + + $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + $ok = preg_match('/(([A-Za-z0-9\+\/]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); + + if ($ok !== 1) { + return false; + } + + [ + 2 => $version, + 3 => $width, + 4 => $height, + 6 => $max, + 8 => $crop, + ] = $matches; + + $preview->setMimeType($mimeTypeDetector->detectPath($fileName)); + + $preview->setWidth((int)$width); + $preview->setHeight((int)$height); + $preview->setCropped($crop === 'crop'); + $preview->setMax($max === 'max'); + + if (!empty($version)) { + $preview->setVersion($version); + } + return $preview; + } + + public function getName(): string { + $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + if ($this->isCropped()) { + $path .= '-crop'; + } + if ($this->isMax()) { + $path .= '-max'; + } + + $ext = $this->getExtension(); + $path .= '.' . $ext; + return $path; + } + + public function getExtension(): string { + return match ($this->getMimeType()) { + 'image/png' => 'png', + 'image/gif' => 'gif', + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + default => 'png', + }; + } + + public function setBucketName(string $bucketName): void { + $this->bucketName = $bucketName; + } + + public function setObjectStoreName(string $objectStoreName): void { + $this->objectStoreName = $objectStoreName; + } + + public function setVersion(?string $version): void { + $this->version = $version; + } + + public function getMimeType(): string { + return $this->mimetype; + } + + public function setMimeType(string $mimeType): void { + $this->mimetype = $mimeType; + } + + public function getSourceMimeType(): string { + return $this->sourceMimetype; + } + + public function setSourceMimeType(string $mimeType): void { + $this->sourceMimetype = $mimeType; + } +} diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php new file mode 100644 index 0000000000000..e6ca2e720f358 --- /dev/null +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -0,0 +1,202 @@ + + */ +class PreviewMapper extends QBMapper { + + private const TABLE_NAME = 'previews'; + private const LOCATION_TABLE_NAME = 'preview_locations'; + private const VERSION_TABLE_NAME = 'preview_versions'; + + public function __construct( + IDBConnection $db, + private readonly IMimeTypeLoader $mimeTypeLoader, + ) { + parent::__construct($db, self::TABLE_NAME, Preview::class); + } + + protected function mapRowToEntity(array $row): Entity { + $row['mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['mimetype_id']); + $row['source_mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['source_mimetype_id']); + + return parent::mapRowToEntity($row); + } + + #[Override] + public function insert(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->insert(self::VERSION_TABLE_NAME) + ->values([ + 'version' => $preview->getVersion(), + 'file_id' => $preview->getFileId(), + ]) + ->executeStatement(); + $entity->setVersionId($qb->getLastInsertId()); + } + return parent::insert($preview); + } + + #[Override] + public function update(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + return parent::update($preview); + } + + #[Override] + public function delete(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::VERSION_TABLE_NAME) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($preview->getFileId()))) + ->andWhere($qb->expr()->eq('version', $qb->createNamedParameter($preview->getVersion()))) + ->executeStatement(); + } + + return parent::delete($entity); + } + + /** + * @return \Generator + * @throws Exception + */ + public function getAvailablePreviewsForFile(int $fileId): \Generator { + $selectQb = $this->db->getQueryBuilder(); + $this->joinLocation($selectQb) + ->where($selectQb->expr()->eq('p.file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); + } + + /** + * @param int[] $fileIds + * @return array + * @throws Exception + */ + public function getAvailablePreviews(array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $this->joinLocation($selectQb) + ->where( + $selectQb->expr()->in('p.file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + ); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + /** + * @return \Generator + */ + public function getByFileId(int $fileId): \Generator { + $selectQb = $this->db->getQueryBuilder(); + $this->joinLocation($selectQb) + ->where($selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); + } + + /** + * @param int[] $previewIds + */ + public function deleteByIds(array $previewIds): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::TABLE_NAME) + ->where($qb->expr()->andX( + $qb->expr()->in('id', $qb->createNamedParameter($previewIds, IQueryBuilder::PARAM_INT_ARRAY)) + ))->executeStatement(); + } + + protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { + return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name', 'v.version') + ->from(self::TABLE_NAME, 'p') + ->leftJoin('p', self::LOCATION_TABLE_NAME, 'l', $qb->expr()->eq( + 'p.location_id', 'l.id' + )) + ->leftJoin('p', self::VERSION_TABLE_NAME, 'v', $qb->expr()->eq( + 'p.version_id', 'v.id' + )); + } + + public function getLocationId(string $bucket, string $objectStore): int { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('id') + ->from(self::LOCATION_TABLE_NAME) + ->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket))) + ->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore))) + ->executeQuery(); + $data = $result->fetchOne(); + if ($data) { + return $data; + } else { + $qb->insert(self::LOCATION_TABLE_NAME) + ->values([ + 'bucket_name' => $qb->createNamedParameter($bucket), + 'object_store_name' => $qb->createNamedParameter($objectStore), + ])->executeStatement(); + return $qb->getLastInsertId(); + } + } + + public function deleteAll(): void { + $delete = $this->db->getQueryBuilder(); + $delete->delete($this->getTableName()); + } + + /** + * @return \Generator + */ + public function getPreviews(int $lastId, int $limit = 1000): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->gt('p.id', $qb->createNamedParameter($lastId, IQueryBuilder::PARAM_INT))) + ->setMaxResults($limit); + return $this->yieldEntities($qb); + + } + + /** + * @param string[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->orX( + ...array_map(function (string $mimeType) use ($qb): string { + return $qb->expr()->eq('source_mimetype_id', $qb->createNamedParameter($this->mimeTypeLoader->getId($mimeType), IQueryBuilder::PARAM_INT)); + }, $mimeTypes) + )); + return $this->yieldEntities($qb); + } +} diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 4a7341896ef6a..82c4ec88363f2 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -6,15 +6,17 @@ */ namespace OC\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\PreviewFile; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\InMemoryFile; use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -31,10 +33,11 @@ class Generator { public function __construct( private IConfig $config, private IPreview $previewManager, - private IAppData $appData, private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, + private PreviewMapper $previewMapper, + private StorageFactory $storageFactory, ) { } @@ -104,32 +107,31 @@ public function generatePreviews(File $file, array $specifications, ?string $mim $mimeType = $file->getMimeType(); } - $previewFolder = $this->getPreviewFolder($file); - // List every existing preview first instead of trying to find them one by one - $previewFiles = $previewFolder->getDirectoryListing(); + [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = ''; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = $file->getPreviewVersion() . '-'; + $previewVersion = $file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that - $maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion); + $maxPreview = $this->getMaxPreview($previews, $file, $mimeType, $previewVersion); $maxPreviewImage = null; // only load the image when we need it if ($maxPreview->getSize() === 0) { - $maxPreview->delete(); + $this->storageFactory->deletePreview($maxPreview); + $this->previewMapper->delete($maxPreview); $this->logger->error('Max preview generated for file {path} has size 0, deleting and throwing exception.', ['path' => $file->getPath()]); throw new NotFoundException('Max preview size 0, invalid!'); } - [$maxWidth, $maxHeight] = $this->getPreviewSize($maxPreview, $previewVersion); + $maxWidth = $maxPreview->getWidth(); + $maxHeight = $maxPreview->getHeight(); if ($maxWidth <= 0 || $maxHeight <= 0) { throw new NotFoundException('The maximum preview sizes are zero or less pixels'); } - $preview = null; - + $previewFile = null; foreach ($specifications as $specification) { $width = $specification['width'] ?? -1; $height = $specification['height'] ?? -1; @@ -148,38 +150,40 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $preview = $maxPreview; + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); continue; } // Try to get a cached preview. Else generate (and store) one try { - try { - $preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion); - } catch (NotFoundException $e) { + $preview = array_find($previews, fn (Preview $preview): bool => $preview->getWidth() === $width + && $preview->getHeight() === $height && $preview->getMimetype() === $maxPreview->getMimetype() + && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); + + if ($preview) { + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); + } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage($maxPreview); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); - // New file, augment our array - $previewFiles[] = $preview; + $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); } - if ($preview->getSize() === 0) { - $preview->delete(); + if ($previewFile->getSize() === 0) { + $previewFile->delete(); throw new NotFoundException('Cached preview size 0, invalid!'); } } - assert($preview !== null); + assert($previewFile !== null); // Free memory being used by the embedded image resource. Without this the image is kept in memory indefinitely. // Garbage Collection does NOT free this memory. We have to do it ourselves. @@ -187,7 +191,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim $maxPreviewImage->destroy(); } - return $preview; + return $previewFile; } /** @@ -289,31 +293,25 @@ public function getNumConcurrentPreviews(string $type): int { } /** - * @param ISimpleFolder $previewFolder - * @param ISimpleFile[] $previewFiles - * @param File $file - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile + * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?string $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. - foreach ($previewFiles as $node) { - $name = $node->getName(); - if (($prefix === '' || str_starts_with($name, $prefix)) && strpos($name, 'max')) { - return $node; + foreach ($previews as $preview) { + if ($preview->isMax() && ($version === $preview->getVersion())) { + return $preview; } } $maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096); $maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096); - return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix); + return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?string $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -322,6 +320,7 @@ private function generateProviderPreview(ISimpleFolder $previewFolder, File $fil } foreach ($providers as $providerClosure) { + $provider = $this->helper->getProvider($providerClosure); if (!($provider instanceof IProviderV2)) { continue; @@ -348,18 +347,25 @@ private function generateProviderPreview(ISimpleFolder $previewFolder, File $fil continue; } - $path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix); try { - if ($preview instanceof IStreamImage) { - return $previewFolder->newFile($path, $preview->resource()); - } else { - return $previewFolder->newFile($path, $preview->data()); - } - } catch (NotPermittedException $e) { + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); + $previewEntry->setSourceMimeType($file->getMimeType()); + $previewEntry->setWidth($preview->width()); + $previewEntry->setHeight($preview->height()); + $previewEntry->setVersion($version); + $previewEntry->setMax($max); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); + $previewEntry->setMimetype($preview->dataMimeType()); + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + return $this->savePreview($previewEntry, $preview); + } catch (NotPermittedException) { throw new NotFoundException(); } - - return $file; } } @@ -367,49 +373,10 @@ private function generateProviderPreview(ISimpleFolder $previewFolder, File $fil } /** - * @param ISimpleFile $file - * @param string $prefix - * @return int[] - */ - private function getPreviewSize(ISimpleFile $file, string $prefix = '') { - $size = explode('-', substr($file->getName(), strlen($prefix))); - return [(int)$size[0], (int)$size[1]]; - } - - /** - * @param int $width - * @param int $height - * @param bool $crop - * @param bool $max - * @param string $mimeType - * @param string $prefix - * @return string - */ - private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) { - $path = $prefix . (string)$width . '-' . (string)$height; - if ($crop) { - $path .= '-crop'; - } - if ($max) { - $path .= '-max'; - } - - $ext = $this->getExtension($mimeType); - $path .= '.' . $ext; - return $path; - } - - - /** - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mode - * @param int $maxWidth - * @param int $maxHeight + * @psalm-param IPreview::MODE_* $mode * @return int[] */ - private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHeight) { + private function calculateSize(int $width, int $height, bool $crop, string $mode, int $maxWidth, int $maxHeight): array { /* * If we are not cropping we have to make sure the requested image * respects the aspect ratio of the original. @@ -492,14 +459,14 @@ private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHei * @throws \InvalidArgumentException if the preview would be invalid (in case the original image is invalid) */ private function generatePreview( - ISimpleFolder $previewFolder, + File $file, IImage $maxPreview, int $width, int $height, bool $crop, int $maxWidth, int $maxHeight, - string $prefix, + ?string $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -535,82 +502,55 @@ private function generatePreview( self::unguardWithSemaphore($sem); } - - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix); - try { - if ($cacheResult) { - return $previewFolder->newFile($path, $preview->data()); - } else { - return new InMemoryFile($path, $preview->data()); - } - } catch (NotPermittedException $e) { - throw new NotFoundException(); - } - return $file; - } - - /** - * @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing() - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile - * - * @throws NotFoundException - */ - private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) { - $path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix); - foreach ($files as $file) { - if ($file->getName() === $path) { - $this->logger->debug('Found cached preview: {path}', ['path' => $path]); - return $file; - } + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); + $previewEntry->setWidth($width); + $previewEntry->setSourceMimeType($file->getMimeType()); + $previewEntry->setHeight($height); + $previewEntry->setVersion($version); + $previewEntry->setMax(false); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); + $previewEntry->setMimeType($preview->dataMimeType()); + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + if ($cacheResult) { + $previewEntry = $this->savePreview($previewEntry, $preview); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); + } else { + return new InMemoryFile($previewEntry->getName(), $preview->data()); } - throw new NotFoundException(); } /** - * Get the specific preview folder for this file - * - * @param File $file - * @return ISimpleFolder - * * @throws InvalidPathException * @throws NotFoundException * @throws NotPermittedException + * @throws \OCP\DB\Exception */ - private function getPreviewFolder(File $file) { - // Obtain file id outside of try catch block to prevent the creation of an existing folder - $fileId = (string)$file->getId(); + public function savePreview(Preview $previewEntry, IImage $preview): Preview { + $previewEntry = $this->previewMapper->insert($previewEntry); + // we need to save to DB first try { - $folder = $this->appData->getFolder($fileId); - } catch (NotFoundException $e) { - $folder = $this->appData->newFolder($fileId); - } - - return $folder; - } - - /** - * @param string $mimeType - * @return null|string - * @throws \InvalidArgumentException - */ - private function getExtension($mimeType) { - switch ($mimeType) { - case 'image/png': - return 'png'; - case 'image/jpeg': - return 'jpg'; - case 'image/webp': - return 'webp'; - case 'image/gif': - return 'gif'; - default: - throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); + if ($preview instanceof IStreamImage) { + $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); + } else { + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $preview->data()); + rewind($stream); + $size = $this->storageFactory->writePreview($previewEntry, $stream); + } + if (!$size) { + throw new \RuntimeException('Unable to write preview file'); + } + } catch (\Exception $e) { + $this->previewMapper->delete($previewEntry); + throw $e; } + $previewEntry->setSize($size); + return $this->previewMapper->update($previewEntry); } } diff --git a/lib/private/Preview/GeneratorHelper.php b/lib/private/Preview/GeneratorHelper.php index e914dcc20020b..7114a412e36ea 100644 --- a/lib/private/Preview/GeneratorHelper.php +++ b/lib/private/Preview/GeneratorHelper.php @@ -45,11 +45,7 @@ public function getThumbnail(IProviderV2 $provider, File $file, $maxWidth, $maxH return $provider->getThumbnail($file, $maxWidth, $maxHeight) ?? false; } - /** - * @param ISimpleFile $maxPreview - * @return IImage - */ - public function getImage(ISimpleFile $maxPreview) { + public function getImage(ISimpleFile $maxPreview): IImage { $image = new OCPImage(); $image->loadFromData($maxPreview->getContent()); return $image; diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php new file mode 100644 index 0000000000000..8d30ae8a40212 --- /dev/null +++ b/lib/private/Preview/PreviewService.php @@ -0,0 +1,109 @@ +storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + + /** + * Get storageId and fileIds for which we have at least one preview. + * + * @return \Generator + */ + public function getAvailableFileIds(): \Generator { + $maxQb = $this->connection->getQueryBuilder(); + $maxQb->select($maxQb->func()->max('id')) + ->from($this->previewMapper->getTableName()) + ->groupBy('file_id'); + + $qb = $this->connection->getQueryBuilder(); + $qb->select('file_id', 'storage_id') + ->from($this->previewMapper->getTableName()) + ->where($qb->expr()->in('id', $qb->createFunction($maxQb->getSQL()))); + + $result = $qb->executeQuery(); + + $lastStorageId = -1; + /** @var int[] $fileIds */ + $fileIds = []; + + // Previews next to each others in the database are likely in the same storage, so group them + while ($row = $result->fetch()) { + if ($lastStorageId !== $row['storage_id']) { + if ($lastStorageId !== -1) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + $fileIds = []; + } + $lastStorageId = (int)$row['storage_id']; + } + $fileIds[] = (int)$row['file_id']; + } + + if (count($fileIds) > 0) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + } + } + + /** + * @return \Generator + */ + public function getAvailablePreviewsForFile(int $fileId): \Generator { + return $this->previewMapper->getAvailablePreviewsForFile($fileId); + } + + /** + * @param string[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + return $this->previewMapper->getPreviewsForMimeTypes($mimeTypes); + } + + public function deleteAll(): void { + $lastId = 0; + while (true) { + $previews = $this->previewMapper->getPreviews($lastId, 1000); + $i = 0; + foreach ($previews as $preview) { + $this->deletePreview($preview); + $i++; + $lastId = $preview->getId(); + } + + if ($i !== 1000) { + break; + } + } + } + + /** + * @param int[] $fileIds + * @return array + */ + public function getAvailablePreviews(array $fileIds): array { + return $this->previewMapper->getAvailablePreviews($fileIds); + } +} diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php new file mode 100644 index 0000000000000..1d6b128f8f022 --- /dev/null +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -0,0 +1,53 @@ +instanceId = $this->config->getSystemValueString('instanceid'); + $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); + } + + #[Override] + public function writePreview(Preview $preview, mixed $stream): int { + $previewPath = $this->constructPath($preview); + $this->createParentFiles($previewPath); + return file_put_contents($previewPath, $stream); + } + + #[Override] + public function readPreview(Preview $preview): mixed { + $previewPath = $this->constructPath($preview); + $resource = @fopen($previewPath, 'r'); + if ($resource === false) { + throw new NotFoundException('Unable to open preview stream at ' . $previewPath); + } + return $resource; + } + + #[Override] + public function deletePreview(Preview $preview): void { + $previewPath = $this->constructPath($preview); + if (!@unlink($previewPath) && is_file($previewPath)) { + throw new NotPermittedException('Unable to delete preview at ' . $previewPath); + } + } + + public function getPreviewRootFolder(): string { + return $this->rootFolder . '/appdata_' . $this->instanceId . '/preview/'; + } + + private function constructPath(Preview $preview): string { + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + } + + private function createParentFiles(string $path): void { + $dirname = dirname($path); + @mkdir($dirname, recursive: true); + if (!is_dir($dirname)) { + throw new NotPermittedException("Unable to create directory '$dirname'"); + } + } + + #[Override] + public function migratePreview(Preview $preview, SimpleFile $file): void { + // legacy flat directory + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); + if (!file_exists($sourcePath)) { + return; + } + + $destinationPath = $this->constructPath($preview); + if (file_exists($destinationPath)) { + @unlink($sourcePath); // We already have a new preview, just delete the old one + return; + } + + $this->createParentFiles($destinationPath); + $ok = rename($sourcePath, $destinationPath); + if (!$ok) { + throw new LogicException('Failed to move ' . $sourcePath . ' to ' . $destinationPath); + } + } + + #[Override] + public function scan(): int { + $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); + + $scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder()); + $previewsFound = 0; + foreach (new RecursiveIteratorIterator($scanner) as $file) { + if ($file->isFile()) { + $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector); + if ($preview === false) { + $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); + continue; + } + try { + $preview->setSize($file->getSize()); + $preview->setMtime($file->getMtime()); + $preview->setEncrypted(false); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('storage', 'etag', 'mimetype') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) + ->setMaxResults(1) + ->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + // original file is deleted + @unlink($file->getRealPath()); + continue; + } + + if ($checkForFileCache) { + $relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath()); + $rowAffected = $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath)))) + ->executeStatement(); + if ($rowAffected > 0) { + $this->deleteParentsFromFileCache(dirname($relativePath)); + } + } + + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setSourceMimetype($result[0]['mimetype']); + + // try to insert, if that fails the preview is already in the DB + $this->previewMapper->insert($preview); + + // Move old flat preview to new format + $dirName = str_replace($this->getPreviewRootFolder(), '', $file->getPath()); + if (preg_match('/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9]+/', $dirName) !== 1) { + $previewPath = $this->constructPath($preview); + $this->createParentFiles($previewPath); + $ok = rename($file->getRealPath(), $previewPath); + if (!$ok) { + throw new LogicException('Failed to move ' . $file->getRealPath() . ' to ' . $previewPath); + } + } + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + $previewsFound++; + } + } + + return $previewsFound; + } + + private function deleteParentsFromFileCache(string $dirname): void { + $qb = $this->connection->getQueryBuilder(); + + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname)))) + ->setMaxResults(1) + ->runAcrossAllShards() + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + return; + } + + $this->connection->beginTransaction(); + + $parentId = $result[0]['parent']; + $fileId = $result[0]['fileid']; + $storage = $result[0]['storage']; + + try { + while (true) { + $qb = $this->connection->getQueryBuilder(); + $childs = $qb->select('fileid', 'path', 'storage') + ->from('filecache') + ->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (!empty($childs)) { + break; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $result[0]['storage']) + ->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId))) + ->setMaxResults(1) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + break; + } + + $fileId = $parentId; + $parentId = $result[0]['parent']; + } + } finally { + $this->connection->commit(); + } + } +} diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php new file mode 100644 index 0000000000000..0c94071d28e39 --- /dev/null +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -0,0 +1,179 @@ +> + */ + private array $objectStoreCache = []; + + private bool $isMultibucketPreviewDistributionEnabled; + + public function __construct( + private readonly PrimaryObjectStoreConfig $objectStoreConfig, + IConfig $config, + readonly private PreviewMapper $previewMapper, + ) { + $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); + } + + #[Override] + public function writePreview(Preview $preview, mixed $stream): int { + $size = 0; + $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { + $size = $writtenSize; + }); + + [ + 'urn' => $urn, + 'store' => $store, + ] = $this->getObjectStoreInfoForNewPreview($preview); + + try { + $store->writeObject($urn, $countStream); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to save preview to object store', previous: $exception); + } + return $size; + } + + #[Override] + public function readPreview(Preview $preview): mixed { + [ + 'urn' => $urn, + 'store' => $store, + ] = $this->getObjectStoreInfoForExistingPreview($preview); + + try { + return $store->readObject($urn); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to read preview from object store with urn:' . $urn, previous: $exception); + } + } + + #[Override] + public function deletePreview(Preview $preview): void { + [ + 'urn' => $urn, + 'store' => $store, + ] = $this->getObjectStoreInfoForExistingPreview($preview); + + try { + $store->deleteObject($urn); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to delete preview from object store', previous: $exception); + } + } + + #[Override] + public function migratePreview(Preview $preview, SimpleFile $file): void { + // Just set the Preview::bucket and Preview::objectStore + $this->getObjectStoreInfoForNewPreview($preview, migration: true); + $this->previewMapper->update($preview); + } + + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreInfoForExistingPreview(Preview $preview): array { + assert(!empty($preview->getObjectStoreName())); + assert(!empty($preview->getBucketName())); + + $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); + $config['arguments']['bucket'] = $preview->getBucketName(); + $objectStoreName = $preview->getObjectStoreName(); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } + + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreInfoForNewPreview(Preview $preview, bool $migration = false): array { + // When migrating old previews, use the 'root' object store configuration + $config = $this->objectStoreConfig->getObjectStoreConfiguration($migration ? 'root' : 'preview'); + $objectStoreName = $this->objectStoreConfig->resolveAlias($migration ? 'root' : 'preview'); + + $bucketName = $config['arguments']['bucket']; + if ($config['arguments']['multibucket']) { + if ($this->isMultibucketPreviewDistributionEnabled) { + // Spread the previews on different buckets depending on their corresponding fileId + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $bucketName .= '-preview-' . $bucketNumber; + } else { + // Put all previews in the root (0) bucket + $bucketName .= '0'; + } + } + $config['arguments']['bucket'] = $bucketName; + + // Get the locationId corresponding to the bucketName and objectStoreName, this will create + // a new one, if no matching location is found in the DB. + $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); + $preview->setLocationId($locationId); + $preview->setObjectStoreName($objectStoreName); + $preview->setBucketName($bucketName); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } + + private function getObjectStore(string $objectStoreName, array $config): IObjectStore { + $bucketName = $config['arguments']['bucket']; + + if (!isset($this->objectStoreCache[$objectStoreName])) { + $this->objectStoreCache[$objectStoreName] = []; + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); + } elseif (!isset($this->objectStoreCache[$objectStoreName][$bucketName])) { + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); + } + + return $this->objectStoreCache[$objectStoreName][$bucketName]; + } + + public function getUrn(Preview $preview, array $config): string { + if ($preview->getOldFileId()) { + return ($config['arguments']['objectPrefix'] ?? 'urn:oid:') . $preview->getOldFileId(); + } + if (isset($config['arguments']['objectPrefix'])) { + return ($config['arguments']['objectPrefix'] . 'preview:') . $preview->getId(); + } else { + return 'uri:oid:preview:' . $preview->getId(); + } + } + + #[Override] + public function scan(): int { + return 0; + } +} diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php new file mode 100644 index 0000000000000..64cafb454707a --- /dev/null +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -0,0 +1,81 @@ +preview->getName(); + } + + #[Override] + public function getSize(): int|float { + return $this->preview->getSize(); + } + + #[Override] + public function getETag(): string { + return $this->preview->getEtag(); + } + + #[Override] + public function getMTime(): int { + return $this->preview->getMtime(); + } + + #[Override] + public function getContent(): string { + $stream = $this->storage->readPreview($this->preview); + return stream_get_contents($stream); + } + + #[Override] + public function putContent($data): void { + } + + #[Override] + public function delete(): void { + $this->storage->deletePreview($this->preview); + $this->previewMapper->delete($this->preview); + } + + #[Override] + public function getMimeType(): string { + return $this->preview->getMimetype(); + } + + #[Override] + public function getExtension(): string { + return $this->preview->getExtension(); + } + + #[Override] + public function read() { + return $this->storage->readPreview($this->preview); + } + + #[Override] + public function write() { + return false; + } +} diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php deleted file mode 100644 index 413786539629d..0000000000000 --- a/lib/private/Preview/Storage/Root.php +++ /dev/null @@ -1,74 +0,0 @@ -isMultibucketPreviewDistributionEnabled = $systemConfig->getValue('objectstore.multibucket.preview-distribution', false) === true; - } - - - public function getFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - - try { - return parent::getFolder($internalFolder); - } catch (NotFoundException $e) { - /* - * The new folder structure is not found. - * Lets try the old one - */ - } - - try { - return parent::getFolder($name); - } catch (NotFoundException $e) { - /* - * The old folder structure is not found. - * Lets try the multibucket fallback if available - */ - if ($this->isMultibucketPreviewDistributionEnabled) { - return parent::getFolder('old-multibucket/' . $internalFolder); - } - - // when there is no further fallback just throw the exception - throw $e; - } - } - - public function newFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - return parent::newFolder($internalFolder); - } - - /* - * Do not allow directory listing on this special root - * since it gets to big and time consuming - */ - public function getDirectoryListing(): array { - return []; - } - - public static function getInternalFolder(string $name): string { - return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; - } - - public function getStorageId(): int { - return $this->getAppDataRootFolder()->getStorage()->getCache()->getNumericStorageId(); - } -} diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php new file mode 100644 index 0000000000000..b15031c6a12ed --- /dev/null +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -0,0 +1,65 @@ +getBackend()->writePreview($preview, $stream); + } + + #[Override] + public function readPreview(Preview $preview): mixed { + return $this->getBackend()->readPreview($preview); + } + + #[Override] + public function deletePreview(Preview $preview): void { + $this->getBackend()->deletePreview($preview); + } + + private function getBackend(): IPreviewStorage { + if ($this->backend) { + return $this->backend; + } + + if ($this->objectStoreConfig->hasObjectStore()) { + $this->backend = Server::get(ObjectStorePreviewStorage::class); + } else { + $this->backend = Server::get(LocalPreviewStorage::class); + } + + return $this->backend; + } + + #[Override] + public function migratePreview(Preview $preview, SimpleFile $file): void { + $this->getBackend()->migratePreview($preview, $file); + } + + #[Override] + public function scan(): int { + return $this->getBackend()->scan(); + } +} diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index 21f040d8342e1..9b95e87d2ed03 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -8,11 +8,12 @@ */ namespace OC\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; use OCP\Files\FileInfo; use OCP\Files\Folder; -use OCP\Files\IAppData; use OCP\Files\Node; -use OCP\Files\NotFoundException; +use OCP\IDBConnection; /** * Class Watcher @@ -22,40 +23,44 @@ * Class that will watch filesystem activity and remove previews as needed. */ class Watcher { - /** @var IAppData */ - private $appData; - /** * Watcher constructor. - * - * @param IAppData $appData */ - public function __construct(IAppData $appData) { - $this->appData = $appData; + public function __construct( + private readonly StorageFactory $storageFactory, + private readonly PreviewMapper $previewMapper, + private readonly IDBConnection $connection, + ) { } - public function postWrite(Node $node) { + public function postWrite(Node $node): void { $this->deleteNode($node); } - protected function deleteNode(FileInfo $node) { + protected function deleteNode(FileInfo $node): void { // We only handle files if ($node instanceof Folder) { return; } + $nodeId = $node->getId(); + if (is_null($nodeId)) { + return; + } + + [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$nodeId]); + $this->connection->beginTransaction(); try { - if (is_null($node->getId())) { - return; + foreach ($previews as $preview) { + $this->storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); } - $folder = $this->appData->getFolder((string)$node->getId()); - $folder->delete(); - } catch (NotFoundException $e) { - //Nothing to do + } finally { + $this->connection->commit(); } } - public function versionRollback(array $data) { + public function versionRollback(array $data): void { if (isset($data['node'])) { $this->deleteNode($data['node']); } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 97e9b5e313c19..fb88409287260 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -8,13 +8,14 @@ namespace OC; use OC\AppFramework\Bootstrap\Coordinator; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; @@ -30,7 +31,6 @@ class PreviewManager implements IPreview { protected IConfig $config; protected IRootFolder $rootFolder; - protected IAppData $appData; protected IEventDispatcher $eventDispatcher; private ?Generator $generator = null; private GeneratorHelper $helper; @@ -57,7 +57,6 @@ class PreviewManager implements IPreview { public function __construct( IConfig $config, IRootFolder $rootFolder, - IAppData $appData, IEventDispatcher $eventDispatcher, GeneratorHelper $helper, ?string $userId, @@ -68,7 +67,6 @@ public function __construct( ) { $this->config = $config; $this->rootFolder = $rootFolder; - $this->appData = $appData; $this->eventDispatcher = $eventDispatcher; $this->helper = $helper; $this->userId = $userId; @@ -133,13 +131,14 @@ private function getGenerator(): Generator { $this->generator = new Generator( $this->config, $this, - $this->appData, new GeneratorHelper( $this->rootFolder, $this->config ), $this->eventDispatcher, $this->container->get(LoggerInterface::class), + $this->container->get(PreviewMapper::class), + $this->container->get(StorageFactory::class), ); } return $this->generator; diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 583604515afe1..6a123cc56dd96 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -13,6 +13,7 @@ use OC\Repair\AddCleanupDeletedUsersBackgroundJob; use OC\Repair\AddCleanupUpdaterBackupsJob; use OC\Repair\AddMetadataGenerationJob; +use OC\Repair\AddMovePreviewJob; use OC\Repair\AddRemoveOldTasksBackgroundJob; use OC\Repair\CleanTags; use OC\Repair\CleanUpAbandonedApps; @@ -199,6 +200,7 @@ public static function getRepairSteps(): array { \OCP\Server::get(RemoveLegacyDatadirFile::class), \OCP\Server::get(AddCleanupDeletedUsersBackgroundJob::class), \OCP\Server::get(SanitizeAccountProperties::class), + \OCP\Server::get(AddMovePreviewJob::class), ]; } diff --git a/lib/private/Repair/AddMovePreviewJob.php b/lib/private/Repair/AddMovePreviewJob.php new file mode 100644 index 0000000000000..5edcd64d46e8e --- /dev/null +++ b/lib/private/Repair/AddMovePreviewJob.php @@ -0,0 +1,27 @@ +jobList->add(MovePreviewJob::class); + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index 4445788ec4e24..927d2ce322422 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -49,7 +49,6 @@ use OC\Files\Mount\CacheMountProvider; use OC\Files\Mount\LocalHomeMountProvider; use OC\Files\Mount\ObjectHomeMountProvider; -use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; use OC\Files\Mount\RootMountProvider; use OC\Files\Node\HookConnector; use OC\Files\Node\LazyRoot; @@ -83,9 +82,11 @@ use OC\OCM\Model\OCMProvider; use OC\OCM\OCMDiscoveryService; use OC\OCS\DiscoveryService; +use OC\Preview\Db\PreviewMapper; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; use OC\Preview\MimeIconProvider; +use OC\Preview\Watcher; use OC\Profile\ProfileManager; use OC\Profiler\Profiler; use OC\Remote\Api\ApiFactory; @@ -292,10 +293,6 @@ public function __construct($webRoot, \OC\Config $config) { return new PreviewManager( $c->get(\OCP\IConfig::class), $c->get(IRootFolder::class), - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ), $c->get(IEventDispatcher::class), $c->get(GeneratorHelper::class), $c->get(ISession::class)->get('user_id'), @@ -307,12 +304,11 @@ public function __construct($webRoot, \OC\Config $config) { }); $this->registerAlias(IMimeIconProvider::class, MimeIconProvider::class); - $this->registerService(\OC\Preview\Watcher::class, function (ContainerInterface $c) { - return new \OC\Preview\Watcher( - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ) + $this->registerService(Watcher::class, function (ContainerInterface $c): Watcher { + return new Watcher( + $c->get(\OC\Preview\Storage\StorageFactory::class), + $c->get(PreviewMapper::class), + $c->get(IDBConnection::class), ); }); @@ -789,7 +785,6 @@ public function __construct($webRoot, \OC\Config $config) { $manager->registerHomeProvider(new LocalHomeMountProvider()); $manager->registerHomeProvider(new ObjectHomeMountProvider($objectStoreConfig)); $manager->registerRootProvider(new RootMountProvider($objectStoreConfig, $config)); - $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config)); return $manager; }); diff --git a/lib/private/Setup.php b/lib/private/Setup.php index 15f0a4eb61700..8781127b30afc 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -15,6 +15,7 @@ use OC\Authentication\Token\PublicKeyTokenProvider; use OC\Authentication\Token\TokenCleanupJob; use OC\Core\BackgroundJobs\GenerateMetadataJob; +use OC\Core\BackgroundJobs\MovePreviewJob; use OC\Log\Rotate; use OC\Preview\BackgroundCleanupJob; use OC\TextProcessing\RemoveOldTasksBackgroundJob; @@ -505,6 +506,7 @@ public static function installBackgroundJobs(): void { $jobList->add(RemoveOldTasksBackgroundJob::class); $jobList->add(CleanupDeletedUsers::class); $jobList->add(GenerateMetadataJob::class); + $jobList->add(MovePreviewJob::class); } /** diff --git a/lib/public/AppFramework/Db/QBMapper.php b/lib/public/AppFramework/Db/QBMapper.php index 7fb5b2a9afd8d..d80bb5aec8b97 100644 --- a/lib/public/AppFramework/Db/QBMapper.php +++ b/lib/public/AppFramework/Db/QBMapper.php @@ -84,7 +84,6 @@ public function delete(Entity $entity): Entity { return $entity; } - /** * Creates a new entry in the db from an entity * diff --git a/lib/public/Preview/BeforePreviewFetchedEvent.php b/lib/public/Preview/BeforePreviewFetchedEvent.php index 8ab875070d99c..69cd281ac0260 100644 --- a/lib/public/Preview/BeforePreviewFetchedEvent.php +++ b/lib/public/Preview/BeforePreviewFetchedEvent.php @@ -21,6 +21,7 @@ */ class BeforePreviewFetchedEvent extends \OCP\EventDispatcher\Event { /** + * @param null|IPreview::MODE_FILL|IPreview::MODE_COVER $mode * @since 25.0.1 */ public function __construct( diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 9266b1ac06700..842ae877339ea 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -11,14 +11,13 @@ * Marks files that should keep multiple preview "versions" for the same file id * * Examples of this are files where the storage backend provides versioning, for those - * files, we dont have fileids for the different versions but still need to be able to generate + * files, we don't have fileIds for the different versions but still need to be able to generate * previews for all versions * * @since 17.0.0 */ interface IVersionedPreviewFile { /** - * @return string * @since 17.0.0 */ public function getPreviewVersion(): string; diff --git a/psalm.xml b/psalm.xml index 648b250c38b6e..abea7c5a443ea 100644 --- a/psalm.xml +++ b/psalm.xml @@ -92,6 +92,7 @@ + diff --git a/tests/Core/Command/Preview/RepairTest.php b/tests/Core/Command/Preview/RepairTest.php deleted file mode 100644 index 9b9cde6dd95af..0000000000000 --- a/tests/Core/Command/Preview/RepairTest.php +++ /dev/null @@ -1,153 +0,0 @@ -config = $this->getMockBuilder(IConfig::class) - ->getMock(); - $this->rootFolder = $this->getMockBuilder(IRootFolder::class) - ->getMock(); - $this->logger = $this->getMockBuilder(LoggerInterface::class) - ->getMock(); - $this->iniGetWrapper = $this->getMockBuilder(IniGetWrapper::class) - ->getMock(); - $this->repair = new Repair( - $this->config, - $this->rootFolder, - $this->logger, - $this->iniGetWrapper, - $this->createMock(ILockingProvider::class) - ); - $this->input = $this->createMock(InputInterface::class); - $this->input->expects($this->any()) - ->method('getOption') - ->willReturnCallback(function ($parameter) { - if ($parameter === 'batch') { - return true; - } - return null; - }); - $this->output = $this->getMockBuilder(ConsoleOutput::class) - ->onlyMethods(['section', 'writeln', 'getFormatter']) - ->getMock(); - $self = $this; - - /* We need format method to return a string */ - $outputFormatter = $this->createMock(OutputFormatterInterface::class); - $outputFormatter->method('isDecorated')->willReturn(false); - $outputFormatter->method('format')->willReturnArgument(0); - - $this->output->expects($this->any()) - ->method('getFormatter') - ->willReturn($outputFormatter); - $this->output->expects($this->any()) - ->method('writeln') - ->willReturnCallback(function ($line) use ($self): void { - $self->outputLines .= $line . "\n"; - }); - } - - public static function dataEmptyTest(): array { - /** directoryNames, expectedOutput */ - return [ - [ - [], - 'All previews are already migrated.' - ], - [ - [['name' => 'a'], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['file', 'folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - [ - [['name' => '23'], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - ]; - } - - #[\PHPUnit\Framework\Attributes\DataProvider('dataEmptyTest')] - public function testEmptyExecute($directoryNames, $expectedOutput): void { - $previewFolder = $this->getMockBuilder(Folder::class) - ->getMock(); - $directories = array_map(function ($element) { - $dir = $this->getMockBuilder(Folder::class) - ->getMock(); - $dir->expects($this->any()) - ->method('getName') - ->willReturn($element['name']); - if (isset($element['content'])) { - $list = []; - foreach ($element['content'] as $item) { - if ($item === 'file') { - $list[] = $this->getMockBuilder(Node::class) - ->getMock(); - } elseif ($item === 'folder') { - $list[] = $this->getMockBuilder(Folder::class) - ->getMock(); - } - } - $dir->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($list); - } - return $dir; - }, $directoryNames); - $previewFolder->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($directories); - $this->rootFolder->expects($this->once()) - ->method('get') - ->with('appdata_/preview') - ->willReturn($previewFolder); - - $this->repair->run($this->input, $this->output); - - $this->assertStringContainsString($expectedOutput, $this->outputLines); - } -} diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php deleted file mode 100644 index 9060bf0d5f56d..0000000000000 --- a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php +++ /dev/null @@ -1,95 +0,0 @@ -logger = $this->createMock(LoggerInterface::class); - $this->config = $this->createMock(IConfig::class); - $this->loader = $this->createMock(StorageFactory::class); - - $this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config); - } - - public function testNoMultibucketObjectStorage(): void { - $this->config->expects($this->once()) - ->method('getSystemValue') - ->with('objectstore_multibucket') - ->willReturn(null); - - $this->assertEquals([], $this->provider->getRootMounts($this->loader)); - } - - public function testMultibucketObjectStorage(): void { - $objectstoreConfig = [ - 'class' => S3::class, - 'arguments' => [ - 'bucket' => 'abc', - 'num_buckets' => 64, - 'key' => 'KEY', - 'secret' => 'SECRET', - 'hostname' => 'IP', - 'port' => 'PORT', - 'use_ssl' => false, - 'use_path_style' => true, - ], - ]; - $this->config->expects($this->any()) - ->method('getSystemValue') - ->willReturnCallback(function ($config) use ($objectstoreConfig) { - if ($config === 'objectstore_multibucket') { - return $objectstoreConfig; - } elseif ($config === 'objectstore.multibucket.preview-distribution') { - return true; - } - return null; - }); - $this->config->expects($this->once()) - ->method('getSystemValueString') - ->with('instanceid') - ->willReturn('INSTANCEID'); - - $mounts = $this->provider->getRootMounts($this->loader); - - // 256 mounts for the subfolders and 1 for the fake root - $this->assertCount(257, $mounts); - - // do some sanity checks if they have correct mount point paths - $this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint()); - $this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint()); - // also test the path of the fake bucket - $this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint()); - } -} diff --git a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php index c13b170ca1843..07e58d56f11b9 100644 --- a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php +++ b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php @@ -212,6 +212,7 @@ public function testMultibucketOldConfig() { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -235,6 +236,7 @@ public function testSingleObjectStore() { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -270,6 +272,7 @@ public function testRoot() { $this->setConfig('objectstore', [ 'default' => 'server1', 'root' => 'server2', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ diff --git a/tests/lib/Files/Storage/Storage.php b/tests/lib/Files/Storage/Storage.php index 8110ab6e8cc93..60e696739eed5 100644 --- a/tests/lib/Files/Storage/Storage.php +++ b/tests/lib/Files/Storage/Storage.php @@ -358,6 +358,7 @@ public function testFOpen($fileName): void { $this->assertTrue($this->instance->file_exists($fileName)); $fh = $this->instance->fopen($fileName, 'r'); + $this->assertTrue(is_resource($fh)); $content = stream_get_contents($fh); $this->assertEquals(file_get_contents($textFile), $content); } diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index ab904f2b49933..80df690ad767b 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -9,16 +9,13 @@ use OC\Files\Storage\Temporary; use OC\Preview\BackgroundCleanupJob; -use OC\Preview\Storage\Root; +use OC\Preview\PreviewService; use OC\PreviewManager; -use OC\SystemConfig; use OCP\App\IAppManager; use OCP\AppFramework\Utility\ITimeFactory; -use OCP\Files\AppData\IAppDataFactory; use OCP\Files\File; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; -use OCP\Files\NotFoundException; use OCP\IDBConnection; use OCP\IPreview; use OCP\Server; @@ -42,6 +39,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { private IRootFolder $rootFolder; private IMimeTypeLoader $mimeTypeLoader; private ITimeFactory $timeFactory; + private PreviewService $previewService; protected function setUp(): void { parent::setUp(); @@ -65,6 +63,7 @@ protected function setUp(): void { $this->rootFolder = Server::get(IRootFolder::class); $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); $this->timeFactory = Server::get(ITimeFactory::class); + $this->previewService = Server::get(PreviewService::class); } protected function tearDown(): void { @@ -75,21 +74,18 @@ protected function tearDown(): void { $this->logout(); - parent::tearDown(); - } + foreach ($this->previewService->getAvailablePreviewsForFile(5) as $preview) { + $this->previewService->deletePreview($preview); + } - private function getRoot(): Root { - return new Root( - Server::get(IRootFolder::class), - Server::get(SystemConfig::class) - ); + parent::tearDown(); } private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); $files = []; - for ($i = 0; $i < 11; $i++) { + foreach (range(0, 10) as $i) { $file = $userFolder->newFile($i . '.txt'); $file->putContent('hello world!'); $this->previewManager->getPreview($file); @@ -99,130 +95,50 @@ private function setup11Previews(): array { return $files; } - private function countPreviews(Root $previewRoot, array $fileIds): int { - $i = 0; - - foreach ($fileIds as $fileId) { - try { - $previewRoot->getFolder((string)$fileId); - } catch (NotFoundException $e) { - continue; - } - - $i++; - } - - return $i; + private function countPreviews(PreviewService $previewService, array $fileIds): int { + $previews = $previewService->getAvailablePreviews($fileIds); + return array_reduce($previews, fn (int $result, array $previews) => $result + count($previews), 0); } public function testCleanupSystemCron(): void { $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); - - $root = $this->getRoot(); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, true); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } public function testCleanupAjax(): void { if ($this->connection->getShardDefinition('filecache')) { $this->markTestSkipped('ajax cron is not supported for sharded setups'); - return; } $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $root = $this->getRoot(); - - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, false); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job->run([]); - - $root = $this->getRoot(); - $this->assertSame(1, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); - } - - public function testOldPreviews(): void { - if ($this->connection->getShardDefinition('filecache')) { - $this->markTestSkipped('old previews are not supported for sharded setups'); - return; - } - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - $f1 = $appdata->newFolder('123456781'); - $f1->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder('123456782'); - $f2->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder((string)PHP_INT_MAX - 1); - $f2->newFile('foo.jpg', 'foo'); - - /* - * Cleanup of OldPreviewLocations should only remove numeric folders on AppData level, - * therefore these files should stay untouched. - */ - $appdata->getFolder('/')->newFile('not-a-directory', 'foo'); - $appdata->getFolder('/')->newFile('133742', 'bar'); - - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - // AppData::getDirectoryListing filters all non-folders - $this->assertSame(3, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } - - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->getRoot(), $this->mimeTypeLoader, true); + $this->assertSame(1, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - // Check if the files created above are still present - // Remember: AppData::getDirectoryListing filters all non-folders - $this->assertSame(0, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index edf5418da6ef7..ceaf483a6588c 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -7,125 +7,142 @@ namespace Test\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; +use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; +use OCP\Preview\IVersionedPreviewFile; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; +use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; +use Test\TestCase; -class GeneratorTest extends \Test\TestCase { - /** @var IConfig&\PHPUnit\Framework\MockObject\MockObject */ - private $config; +abstract class VersionedPreviewFile implements IVersionedPreviewFile, File { - /** @var IPreview&\PHPUnit\Framework\MockObject\MockObject */ - private $previewManager; - - /** @var IAppData&\PHPUnit\Framework\MockObject\MockObject */ - private $appData; - - /** @var GeneratorHelper&\PHPUnit\Framework\MockObject\MockObject */ - private $helper; - - /** @var IEventDispatcher&\PHPUnit\Framework\MockObject\MockObject */ - private $eventDispatcher; - - /** @var Generator */ - private $generator; +} - private LoggerInterface&\PHPUnit\Framework\MockObject\MockObject $logger; +class GeneratorTest extends TestCase { + private IConfig&MockObject $config; + private IPreview&MockObject $previewManager; + private GeneratorHelper&MockObject $helper; + private IEventDispatcher&MockObject $eventDispatcher; + private Generator $generator; + private LoggerInterface&MockObject $logger; + private StorageFactory&MockObject $storageFactory; + private PreviewMapper&MockObject $previewMapper; protected function setUp(): void { parent::setUp(); $this->config = $this->createMock(IConfig::class); $this->previewManager = $this->createMock(IPreview::class); - $this->appData = $this->createMock(IAppData::class); $this->helper = $this->createMock(GeneratorHelper::class); $this->eventDispatcher = $this->createMock(IEventDispatcher::class); $this->logger = $this->createMock(LoggerInterface::class); + $this->previewMapper = $this->createMock(PreviewMapper::class); + $this->storageFactory = $this->createMock(StorageFactory::class); $this->generator = new Generator( $this->config, $this->previewManager, - $this->appData, $this->helper, $this->eventDispatcher, $this->logger, + $this->previewMapper, + $this->storageFactory, ); } - public function testGetCachedPreview(): void { - $file = $this->createMock(File::class); + private function getFile(int $fileId, string $mimeType, bool $hasVersion = false): File { + $mountPoint = $this->createMock(IMountPoint::class); + $mountPoint->method('getNumericStorageId')->willReturn(42); + if ($hasVersion) { + $file = $this->createMock(VersionedPreviewFile::class); + $file->method('getPreviewVersion')->willReturn('abc'); + } else { + $file = $this->createMock(File::class); + } $file->method('isReadable') ->willReturn(true); $file->method('getMimeType') - ->willReturn('myMimeType'); + ->willReturn($mimeType); $file->method('getId') - ->willReturn(42); + ->willReturn($fileId); + $file->method('getMountPoint') + ->willReturn($mountPoint); + return $file; + } + + #[TestWith([true])] + #[TestWith([false])] + public function testGetCachedPreview(bool $hasPreview): void { + $file = $this->getFile(42, 'myMimeType', $hasPreview); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('1000-1000-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); - $previewFile->method('getName')->willReturn('256-256.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $previewFile]); + $maxPreview = new Preview(); + $maxPreview->setWidth(1000); + $maxPreview->setHeight(1000); + $maxPreview->setMax(true); + $maxPreview->setSize(1000); + $maxPreview->setCropped(false); + $maxPreview->setStorageId(1); + $maxPreview->setVersion($hasPreview ? 'abc' : null); + $maxPreview->setMimeType('image/png'); + + $previewFile = new Preview(); + $previewFile->setWidth(256); + $previewFile->setHeight(256); + $previewFile->setMax(false); + $previewFile->setSize(1000); + $previewFile->setVersion($hasPreview ? 'abc' : null); + $previewFile->setCropped(false); + $previewFile->setStorageId(1); + $previewFile->setMimeType('image/png'); + + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame($hasPreview ? 'abc-256-256.png' : '256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } - public function testGetNewPreview(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + #[TestWith([true])] + #[TestWith([false])] + public function testGetNewPreview(bool $hasVersion): void { + $file = $this->getFile(42, 'myMimeType', $hasVersion); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willThrowException(new NotFoundException()); - - $this->appData->method('newFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); - $this->config->method('getSystemValue') + $this->config->method('getSystemValueString') ->willReturnCallback(function ($key, $default) { return $default; }); @@ -175,7 +192,7 @@ public function testGetNewPreview(): void { $image->method('dataMimeType')->willReturn('image/png'); $this->helper->method('getThumbnail') - ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image) { + ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image): false|IImage { if ($provider === $validProvider) { return $image; } else { @@ -186,29 +203,39 @@ public function testGetNewPreview(): void { $image->method('data') ->willReturn('my data'); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName')->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType')->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); - - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); - - $previewFolder->method('getDirectoryListing') - ->willReturn([]); - $previewFolder->method('newFile') - ->willReturnMap([ - ['2048-2048-max.png', 'my data', $maxPreview], - ['256-256.png', 'my resized data', $previewFile], - ]); - - $previewFolder->method('getFile') - ->with($this->equalTo('256-256.png')) - ->willThrowException(new NotFoundException()); + $this->previewMapper->method('insert') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->storageFactory->method('writePreview') + ->willReturnCallback(function (Preview $preview, mixed $data) use ($hasVersion): int { + $data = stream_get_contents($data); + if ($hasVersion) { + switch ($preview->getName()) { + case 'abc-2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case 'abc-256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + } else { + switch ($preview->getName()) { + case '2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case '256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + } + $this->fail('file name is wrong:' . $preview->getName()); + }); $image = $this->getMockImage(2048, 2048, 'my resized data'); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); $this->eventDispatcher->expects($this->once()) @@ -216,39 +243,32 @@ public function testGetNewPreview(): void { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame($hasVersion ? 'abc-256-256.png' : '256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } public function testInvalidMimeType(): void { $this->expectException(NotFoundException::class); - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'invalidType'); $this->previewManager->method('isMimeSupported') ->with('invalidType') ->willReturn(false); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(null); + $maxPreview->setMimetype('image/png'); - $previewFolder->method('getFile') - ->with($this->equalTo('1024-512-crop.png')) - ->willThrowException(new NotFoundException()); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -258,31 +278,31 @@ public function testInvalidMimeType(): void { } public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); - - - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $preview->method('getName')->willReturn('1024-512-crop.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $preview]); + $file = $this->getFile(42, 'myMimeType'); + + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); + + $previewFile = new Preview(); + $previewFile->setWidth(1024); + $previewFile->setHeight(512); + $previewFile->setMax(false); + $previewFile->setSize(1000); + $previewFile->setCropped(true); + $previewFile->setVersion(null); + $previewFile->setMimeType('image/png'); + + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->previewManager->expects($this->never()) ->method('isMimeSupported'); @@ -292,25 +312,15 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void ->with(new BeforePreviewFetchedEvent($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType')); $result = $this->generator->getPreview($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType'); - $this->assertSame($preview, $result); + $this->assertSame('1024-512-crop.png', $result->getName()); } public function testNoProvider(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $previewFolder->method('getDirectoryListing') - ->willReturn([]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); $this->previewManager->method('getProviders') ->willReturn([]); @@ -323,7 +333,7 @@ public function testNoProvider(): void { $this->generator->getPreview($file, 100, 100); } - private function getMockImage($width, $height, $data = null) { + private function getMockImage(int $width, int $height, string $data = '') { $image = $this->createMock(IImage::class); $image->method('height')->willReturn($width); $image->method('width')->willReturn($height); @@ -380,65 +390,52 @@ public static function dataSize(): array { ]; } - /** - * - * @param int $maxX - * @param int $maxY - * @param int $reqX - * @param int $reqY - * @param bool $crop - * @param string $mode - * @param int $expectedX - * @param int $expectedY - */ - #[\PHPUnit\Framework\Attributes\DataProvider('dataSize')] - public function testCorrectSize($maxX, $maxY, $reqX, $reqY, $crop, $mode, $expectedX, $expectedY): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + #[DataProvider('dataSize')] + public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $crop, string $mode, int $expectedX, int $expectedY): void { + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $maxPreview = new Preview(); + $maxPreview->setWidth($maxX); + $maxPreview->setHeight($maxY); + $maxPreview->setMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn($maxX . '-' . $maxY . '-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); + $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimeType(), 'image/png'); - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $filename = $expectedX . '-' . $expectedY; if ($crop) { $filename .= '-crop'; } $filename .= '.png'; - $previewFolder->method('getFile') - ->with($this->equalTo($filename)) - ->willThrowException(new NotFoundException()); $image = $this->getMockImage($maxX, $maxY); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $previewFolder->method('newFile') - ->with($this->equalTo($filename)) - ->willReturn($preview); + $this->previewMapper->method('insert') + ->willReturnCallback(function (Preview $preview) use ($filename): Preview { + $this->assertSame($preview->getName(), $filename); + return $preview; + }); + + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->storageFactory->method('writePreview') + ->willReturn(1000); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -446,9 +443,9 @@ public function testCorrectSize($maxX, $maxY, $reqX, $reqY, $crop, $mode, $expec $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview, $result); + $this->assertSame($maxPreview->getName(), $result->getName()); } else { - $this->assertSame($preview, $result); + $this->assertSame($filename, $result->getName()); } } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php new file mode 100644 index 0000000000000..8c9df4274f0f5 --- /dev/null +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -0,0 +1,224 @@ +previewAppData = Server::get(IAppDataFactory::class)->get('preview'); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->config = Server::get(IConfig::class); + $this->appConfig = $this->createMock(IAppConfig::class); + $this->appConfig->expects($this->any()) + ->method('getValueBool') + ->willReturn(false); + $this->appConfig->expects($this->any()) + ->method('setValueBool') + ->willReturn(true); + $this->storageFactory = Server::get(StorageFactory::class); + $this->previewService = Server::get(PreviewService::class); + $this->db = Server::get(IDBConnection::class); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); + + $qb = $this->db->getQueryBuilder(); + $qb->insert('filecache') + ->values([ + 'fileid' => $qb->createNamedParameter(5), + 'storage' => $qb->createNamedParameter(1), + 'path' => $qb->createNamedParameter('test/abc'), + 'path_hash' => $qb->createNamedParameter(md5('test')), + 'parent' => $qb->createNamedParameter(0), + 'name' => $qb->createNamedParameter('abc'), + 'mimetype' => $qb->createNamedParameter(42), + 'size' => $qb->createNamedParameter(1000), + 'mtime' => $qb->createNamedParameter(1000), + 'storage_mtime' => $qb->createNamedParameter(1000), + 'encrypted' => $qb->createNamedParameter(0), + 'unencrypted_size' => $qb->createNamedParameter(0), + 'etag' => $qb->createNamedParameter('abcdefg'), + 'permissions' => $qb->createNamedParameter(0), + 'checksum' => $qb->createNamedParameter('abcdefg'), + ])->executeStatement(); + + $this->mimeTypeDetector = $this->createMock(IMimeTypeDetector::class); + $this->mimeTypeDetector->method('detectPath')->willReturn('image/png'); + $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader->method('getId')->with('image/png')->willReturn(42); + $this->mimeTypeLoader->method('getMimetypeById')->with(42)->willReturn('image/png'); + $this->logger = $this->createMock(LoggerInterface::class); + } + + public function tearDown(): void { + foreach ($this->previewAppData->getDirectoryListing() as $folder) { + $folder->delete(); + } + $this->previewService->deleteAll(); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); + } + + #[TestDox('Test the migration from the legacy flat hierarchy to the new database format')] + public function testMigrationLegacyPath(): void { + $folder = $this->previewAppData->newFolder('5'); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->config, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, + Server::get(IAppDataFactory::class), + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + } + + private static function getInternalFolder(string $name): string { + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + public function testMigrationPath(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->config, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + public function testMigrationPathWithVersion(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); + // No version + $folder->newFile('128-128-crop.png', 'abcdefg'); + $folder->newFile('256-256-max.png', 'abcdefg'); + $folder->newFile('128-128.png', 'abcdefg'); + + // Version 1000 + $folder->newFile('1000-128-128-crop.png', 'abcdefg'); + $folder->newFile('1000-256-256-max.png', 'abcdefg'); + $folder->newFile('1000-128-128.png', 'abcdefg'); + + // Version 1001 + $folder->newFile('1001-128-128-crop.png', 'abcdefg'); + $folder->newFile('1001-256-256-max.png', 'abcdefg'); + $folder->newFile('1001-128-128.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); + $this->assertEquals(9, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->config, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)); + $this->assertEquals(9, count($previews)); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + + $nameVersionMapping = []; + foreach ($previews as $preview) { + $nameVersionMapping[$preview->getName($this->mimeTypeLoader)] = $preview->getVersion(); + } + + $this->assertEquals([ + '1000-128-128-crop.png' => 1000, + '1000-128-128.png' => 1000, + '1000-256-256-max.png' => 1000, + '1001-128-128-crop.png' => 1001, + '1001-128-128.png' => 1001, + '1001-256-256-max.png' => 1001, + '128-128-crop.png' => null, + '128-128.png' => null, + '256-256-max.png' => null, + ], $nameVersionMapping); + } +} diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php new file mode 100644 index 0000000000000..8e27a642473d0 --- /dev/null +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -0,0 +1,82 @@ +previewMapper = Server::get(PreviewMapper::class); + $this->connection = Server::get(IDBConnection::class); + } + + public function testGetAvailablePreviews(): void { + // Empty + $this->assertEquals([], $this->previewMapper->getAvailablePreviews([])); + + // No preview available + $this->assertEquals([42 => []], $this->previewMapper->getAvailablePreviews([42])); + + $this->createPreviewForFileId(42); + $previews = $this->previewMapper->getAvailablePreviews([42]); + $this->assertNotEmpty($previews[42]); + $this->assertNull($previews[42][0]->getLocationId()); + $this->assertNull($previews[42][0]->getBucketName()); + $this->assertNull($previews[42][0]->getObjectStoreName()); + + $this->createPreviewForFileId(43, 2); + $previews = $this->previewMapper->getAvailablePreviews([43]); + $this->assertNotEmpty($previews[43]); + $this->assertEquals('preview-2', $previews[43][0]->getBucketName()); + $this->assertEquals('default', $previews[43][0]->getObjectStoreName()); + } + + private function createPreviewForFileId(int $fileId, ?int $bucket = null): void { + $locationId = null; + if ($bucket) { + $qb = $this->connection->getQueryBuilder(); + $qb->insert('preview_locations') + ->values([ + 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), + 'object_store_name' => $qb->createNamedParameter('default'), + ]); + $qb->executeStatement(); + $locationId = $qb->getLastInsertId(); + } + $preview = new Preview(); + $preview->setFileId($fileId); + $preview->setStorageId(1); + $preview->setCropped(true); + $preview->setMax(true); + $preview->setWidth(100); + $preview->setSourceMimeType('image/jpeg'); + $preview->setHeight(100); + $preview->setSize(100); + $preview->setMtime(time()); + $preview->setMimetype('image/jpeg'); + $preview->setEtag('abcdefg'); + + if ($locationId !== null) { + $preview->setLocationId($locationId); + } + $this->previewMapper->insert($preview); + } +} diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php new file mode 100644 index 0000000000000..f3f9c8ae895d3 --- /dev/null +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -0,0 +1,60 @@ +previewService = Server::get(PreviewService::class); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->previewService->deleteAll(); + } + + public function tearDown(): void { + $this->previewService->deleteAll(); + } + + public function testGetAvailableFileIds(): void { + foreach (range(1, 20) as $i) { + $preview = new Preview(); + $preview->setFileId($i % 10); + $preview->setStorageId(1); + $preview->setWidth($i); + $preview->setHeight($i); + $preview->setMax(true); + $preview->setSourceMimeType('image/jpeg'); + $preview->setCropped(true); + $preview->setEncrypted(false); + $preview->setMimetype('image/jpeg'); + $preview->setEtag('abc'); + $preview->setMtime((new \DateTime())->getTimestamp()); + $preview->setSize(0); + $this->previewMapper->insert($preview); + } + + $files = iterator_to_array($this->previewService->getAvailableFileIds()); + $this->assertCount(1, $files); + $this->assertCount(10, $files[0]['fileIds']); + } +} diff --git a/version.php b/version.php index d244b03ab24ae..b4a6c28365597 100644 --- a/version.php +++ b/version.php @@ -9,7 +9,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patch level // when updating major/minor version number. -$OC_Version = [33, 0, 0, 0]; +$OC_Version = [33, 0, 0, 1]; // The human-readable string $OC_VersionString = '33.0.0 dev';