From 18fbacdd8d519e88e8cc53438a6209428f90085d Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Wed, 20 Aug 2025 17:34:07 +0200 Subject: [PATCH 01/14] perf(preview): Split preview data to new table The new oc_previews table is optimized for storing previews and should decrease significantly the space taken by previews in the filecache table. This attend to reuse the IObjectStore abstraction over S3/Swift/Azure but currently only support one single bucket configuration. Signed-off-by: Carl Schwan --- .../Version33000Date20250819110529.php | 48 ++++ lib/composer/composer/LICENSE | 2 - lib/composer/composer/autoload_classmap.php | 8 + lib/composer/composer/autoload_static.php | 8 + .../ObjectStore/PrimaryObjectStoreConfig.php | 2 +- lib/private/Preview/Db/Preview.php | 105 ++++++++ lib/private/Preview/Db/PreviewMapper.php | 66 +++++ lib/private/Preview/Generator.php | 226 ++++++++---------- .../Preview/Storage/IPreviewStorage.php | 22 ++ .../Preview/Storage/LocalPreviewStorage.php | 43 ++++ .../Storage/ObjectStorePreviewStorage.php | 50 ++++ lib/private/Preview/Storage/PreviewFile.php | 90 +++++++ .../Preview/Storage/StorageFactory.php | 44 ++++ lib/private/PreviewManager.php | 4 + lib/public/IPreview.php | 5 + 15 files changed, 596 insertions(+), 127 deletions(-) create mode 100644 core/Migrations/Version33000Date20250819110529.php create mode 100644 lib/private/Preview/Db/Preview.php create mode 100644 lib/private/Preview/Db/PreviewMapper.php create mode 100644 lib/private/Preview/Storage/IPreviewStorage.php create mode 100644 lib/private/Preview/Storage/LocalPreviewStorage.php create mode 100644 lib/private/Preview/Storage/ObjectStorePreviewStorage.php create mode 100644 lib/private/Preview/Storage/PreviewFile.php create mode 100644 lib/private/Preview/Storage/StorageFactory.php diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php new file mode 100644 index 0000000000000..fcc008b428381 --- /dev/null +++ b/core/Migrations/Version33000Date20250819110529.php @@ -0,0 +1,48 @@ +hasTable('previews')) { + $table = $schema->createTable('previews'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); + } + + return $schema; + } +} diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index f27399a042d95..62ecfd8d0046b 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,4 +1,3 @@ - Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index ee77fbd4cda82..80191ae4b7fef 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1528,6 +1528,7 @@ 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1880,6 +1881,8 @@ 'OC\\Preview\\BackgroundCleanupJob' => $baseDir . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => $baseDir . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => $baseDir . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => $baseDir . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => $baseDir . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => $baseDir . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => $baseDir . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => $baseDir . '/lib/private/Preview/GIF.php', @@ -1912,7 +1915,12 @@ 'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 3b18f00da9697..923cf1d5f0b09 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1569,6 +1569,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1921,6 +1922,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\BackgroundCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => __DIR__ . '/../../..' . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => __DIR__ . '/../../..' . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => __DIR__ . '/../../..' . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => __DIR__ . '/../../..' . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => __DIR__ . '/../../..' . '/lib/private/Preview/GIF.php', @@ -1953,7 +1956,12 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 008431b3fbf4f..4a31d2e51f5c9 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix: ?string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php new file mode 100644 index 0000000000000..f2f0587108273 --- /dev/null +++ b/lib/private/Preview/Db/Preview.php @@ -0,0 +1,105 @@ +addType('fileId', Types::INTEGER); + $this->addType('width', Types::INTEGER); + $this->addType('height', Types::INTEGER); + $this->addType('mimetype', Types::INTEGER); + $this->addType('mtime', Types::INTEGER); + $this->addType('size', Types::INTEGER); + $this->addType('isMax', Types::BOOLEAN); + $this->addType('crop', Types::BOOLEAN); + $this->addType('etag', Types::STRING); + $this->addType('version', Types::INTEGER); + } + + public function getName(): string { + $path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + if ($this->getCrop()) { + $path .= '-crop'; + } + if ($this->getIsMax()) { + $path .= '-max'; + } + + $ext = $this->getExtension(); + $path .= '.' . $ext; + return $path; + } + + public function getMimetypeValue(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'image/jpeg', + IPreview::MIMETYPE_PNG => 'image/png', + IPreview::MIMETYPE_WEBP => 'image/webp', + IPreview::MIMETYPE_GIF => 'image/gif', + }; + } + + public function getExtension(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'jpeg', + IPreview::MIMETYPE_PNG => 'png', + IPreview::MIMETYPE_WEBP => 'webp', + IPreview::MIMETYPE_GIF => 'gif', + }; + } +} diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php new file mode 100644 index 0000000000000..5faa508c8e03b --- /dev/null +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -0,0 +1,66 @@ + + */ +class PreviewMapper extends QBMapper { + + private const TABLE_NAME = 'previews'; + + public function __construct(IDBConnection $db) { + parent::__construct($db, self::TABLE_NAME, Preview::class); + } + + /** + * @param int[] $fileIds + * @return array + * @throws Exception + */ + public function getAvailablePreviews(array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + ); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), + $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), + $selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)), + $selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)), + $selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)), + ); + try { + return $this->findEntity($selectQb); + } catch (DoesNotExistException) { + return null; + } + } +} diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 4a7341896ef6a..39e1de54e69d3 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -6,6 +6,10 @@ */ namespace OC\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\PreviewFile; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; use OCP\Files\IAppData; @@ -23,6 +27,7 @@ use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; use Psr\Log\LoggerInterface; +use function Symfony\Component\Translation\t; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -35,6 +40,8 @@ public function __construct( private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, + private PreviewMapper $previewMapper, + private StorageFactory $storageFactory, ) { } @@ -104,25 +111,25 @@ public function generatePreviews(File $file, array $specifications, ?string $mim $mimeType = $file->getMimeType(); } - $previewFolder = $this->getPreviewFolder($file); - // List every existing preview first instead of trying to find them one by one - $previewFiles = $previewFolder->getDirectoryListing(); + [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = ''; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = $file->getPreviewVersion() . '-'; + $previewVersion = (int)$file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that - $maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion); + $maxPreview = $this->getMaxPreview($previews, $file, $mimeType, $previewVersion); $maxPreviewImage = null; // only load the image when we need it if ($maxPreview->getSize() === 0) { - $maxPreview->delete(); + $this->storageFactory->deletePreview($maxPreview); + $this->previewMapper->delete($maxPreview); $this->logger->error('Max preview generated for file {path} has size 0, deleting and throwing exception.', ['path' => $file->getPath()]); throw new NotFoundException('Max preview size 0, invalid!'); } - [$maxWidth, $maxHeight] = $this->getPreviewSize($maxPreview, $previewVersion); + $maxWidth = $maxPreview->getWidth(); + $maxHeight = $maxPreview->getHeight(); if ($maxWidth <= 0 || $maxHeight <= 0) { throw new NotFoundException('The maximum preview sizes are zero or less pixels'); @@ -154,32 +161,40 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // Try to get a cached preview. Else generate (and store) one try { - try { - $preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion); - } catch (NotFoundException $e) { + /** @var ISimpleFile $previewFile */ + $previewFile = null; + // TODO(php8.4) replace by array_find + foreach ($previews as $p) { + if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) { + $previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper); + break; + } + } + + if ($previewFile === null) { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage($maxPreview); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); + $preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); // New file, augment our array - $previewFiles[] = $preview; + //$previews[] = $preview; } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); } - if ($preview->getSize() === 0) { - $preview->delete(); + if ($previewFile->getSize() === 0) { + $previewFile->delete(); throw new NotFoundException('Cached preview size 0, invalid!'); } } - assert($preview !== null); + assert($previewFile !== null); // Free memory being used by the embedded image resource. Without this the image is kept in memory indefinitely. // Garbage Collection does NOT free this memory. We have to do it ourselves. @@ -187,7 +202,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim $maxPreviewImage->destroy(); } - return $preview; + return $previewFile; } /** @@ -289,31 +304,25 @@ public function getNumConcurrentPreviews(string $type): int { } /** - * @param ISimpleFolder $previewFolder - * @param ISimpleFile[] $previewFiles - * @param File $file - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile + * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. - foreach ($previewFiles as $node) { - $name = $node->getName(); - if (($prefix === '' || str_starts_with($name, $prefix)) && strpos($name, 'max')) { - return $node; + foreach ($previews as $preview) { + if ($preview->getIsMax() && ($version == $preview->getVersion())) { + return $preview; } } $maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096); $maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096); - return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix); + return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -348,45 +357,19 @@ private function generateProviderPreview(ISimpleFolder $previewFolder, File $fil continue; } - $path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix); try { - if ($preview instanceof IStreamImage) { - return $previewFolder->newFile($path, $preview->resource()); - } else { - return $previewFolder->newFile($path, $preview->data()); - } + return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version); } catch (NotPermittedException $e) { throw new NotFoundException(); } - - return $file; } } throw new NotFoundException('No provider successfully handled the preview generation'); } - /** - * @param ISimpleFile $file - * @param string $prefix - * @return int[] - */ - private function getPreviewSize(ISimpleFile $file, string $prefix = '') { - $size = explode('-', substr($file->getName(), strlen($prefix))); - return [(int)$size[0], (int)$size[1]]; - } - - /** - * @param int $width - * @param int $height - * @param bool $crop - * @param bool $max - * @param string $mimeType - * @param string $prefix - * @return string - */ - private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) { - $path = $prefix . (string)$width . '-' . (string)$height; + private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string { + $path = ($version ? $version . '-' : '') . $width . '-' . $height; if ($crop) { $path .= '-crop'; } @@ -401,15 +384,10 @@ private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) /** - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mode - * @param int $maxWidth - * @param int $maxHeight + * @psalm-param IPreview::MODE_* $mode * @return int[] */ - private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHeight) { + private function calculateSize(int $width, int $height, bool $crop, string $mode, int $maxWidth, int $maxHeight): array { /* * If we are not cropping we have to make sure the requested image * respects the aspect ratio of the original. @@ -492,14 +470,14 @@ private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHei * @throws \InvalidArgumentException if the preview would be invalid (in case the original image is invalid) */ private function generatePreview( - ISimpleFolder $previewFolder, + File $file, IImage $maxPreview, int $width, int $height, bool $crop, int $maxWidth, int $maxHeight, - string $prefix, + ?int $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -536,62 +514,13 @@ private function generatePreview( } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix); - try { - if ($cacheResult) { - return $previewFolder->newFile($path, $preview->data()); - } else { - return new InMemoryFile($path, $preview->data()); - } - } catch (NotPermittedException $e) { - throw new NotFoundException(); - } - return $file; - } - - /** - * @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing() - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile - * - * @throws NotFoundException - */ - private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) { - $path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix); - foreach ($files as $file) { - if ($file->getName() === $path) { - $this->logger->debug('Found cached preview: {path}', ['path' => $path]); - return $file; - } - } - throw new NotFoundException(); - } - - /** - * Get the specific preview folder for this file - * - * @param File $file - * @return ISimpleFolder - * - * @throws InvalidPathException - * @throws NotFoundException - * @throws NotPermittedException - */ - private function getPreviewFolder(File $file) { - // Obtain file id outside of try catch block to prevent the creation of an existing folder - $fileId = (string)$file->getId(); - - try { - $folder = $this->appData->getFolder($fileId); - } catch (NotFoundException $e) { - $folder = $this->appData->newFolder($fileId); + $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); + if ($cacheResult) { + $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); + } else { + return new InMemoryFile($path, $preview->data()); } - - return $folder; } /** @@ -613,4 +542,53 @@ private function getExtension($mimeType) { throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); } } + + /** + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException + * @throws \OCP\DB\Exception + */ + public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setWidth($width); + $previewEntry->setHeight($height); + $previewEntry->setVersion($version); + $previewEntry->setIsMax($max); + $previewEntry->setCrop($crop); + switch ($preview->dataMimeType()) { + case 'image/jpeg': + $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); + break; + case 'image/gif': + $previewEntry->setMimetype(IPreview::MIMETYPE_GIF); + break; + case 'image/webp': + $previewEntry->setMimetype(IPreview::MIMETYPE_WEBP); + break; + default: + $previewEntry->setMimetype(IPreview::MIMETYPE_PNG); + break; + } + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + + $previewEntry = $this->previewMapper->insert($previewEntry); + + // we need to save to DB first + try { + if ($preview instanceof IStreamImage) { + $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); + } else { + $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); + } + } catch (\Exception $e) { + $this->previewMapper->delete($previewEntry); + throw $e; + } + $previewEntry->setSize($size); + return $this->previewMapper->update($previewEntry); + } } diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php new file mode 100644 index 0000000000000..a989c492c7606 --- /dev/null +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -0,0 +1,22 @@ +constructPath($preview); + ['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath); + $currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY; + mkdir($currentDir); + foreach (explode('/', $dirname) as $suffix) { + $currentDir .= "/$suffix"; + mkdir($currentDir); + } + $file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w"); + return fwrite($file, $stream); + } + + public function readPreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r"); + } + + public function deletePreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + @unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath); + } + + private function constructPath(Preview $preview): string { + return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + } +} diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php new file mode 100644 index 0000000000000..fc9a64ed761f6 --- /dev/null +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -0,0 +1,50 @@ +objectPrefix = $parameters['objectPrefix'] . 'preview:'; + } + } + + public function writePreview(Preview $preview, $stream): false|int { + if (!is_resource($stream)) { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $stream); + rewind($fh); + + $stream = $fh; + } + + $size = 0; + $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { + $size = $writtenSize; + }); + + $this->objectStore->writeObject($this->constructUrn($preview), $countStream); + return $size; + } + + public function readPreview(Preview $preview) { + return $this->objectStore->readObject($this->constructUrn($preview)); + } + + public function deletePreview(Preview $preview) { + return $this->objectStore->deleteObject($this->constructUrn($preview)); + } + + private function constructUrn(Preview $preview): string { + return $this->objectPrefix . $preview->getId(); + } +} diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php new file mode 100644 index 0000000000000..f9e4cc0f59aa0 --- /dev/null +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -0,0 +1,90 @@ +preview->getName(); + } + + /** + * @inheritDoc + */ + public function getSize(): int|float { + return $this->preview->getSize(); + } + + /** + * @inheritDoc + */ + public function getETag(): string { + return $this->preview->getEtag(); + } + + /** + * @inheritDoc + */ + public function getMTime(): int { + return $this->preview->getMtime(); + } + + /** + * @inheritDoc + */ + public function getContent(): string { + $stream = $this->storage->readPreview($this->preview); + return stream_get_contents($stream); + } + + /** + * @inheritDoc + */ + public function putContent($data): void { + } + + /** + * @inheritDoc + */ + public function delete(): void { + $this->storage->deletePreview($this->preview); + $this->previewMapper->delete($this->preview); + } + + /** + * @inheritDoc + */ + public function getMimeType(): string { + return $this->preview->getMimetypeValue(); + } + + /** + * @inheritDoc + */ + public function getExtension(): string { + return $this->preview->getExtension(); + } + + /** + * @inheritDoc + */ + public function read() { + return $this->storage->readPreview($this->preview); + } + + /** + * @inheritDoc + */ + public function write() { + return false; + } +} diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php new file mode 100644 index 0000000000000..158662767742f --- /dev/null +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -0,0 +1,44 @@ +getBackend()->writePreview($preview, $stream); + } + + public function readPreview(Preview $preview) { + return $this->getBackend()->readPreview($preview); + } + + public function deletePreview(Preview $preview) { + $this->getBackend()->deletePreview($preview); + } + + private function getBackend(): IPreviewStorage { + if ($this->backend) { + return $this->backend; + } + + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + + if ($objectStoreConfig) { + $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); + $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); + } else { + $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); + $this->backend = new LocalPreviewStorage($configDataDirectory); + } + + return $this->backend; + } +} diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 97e9b5e313c19..7c8e1b13eb99f 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -8,9 +8,11 @@ namespace OC; use OC\AppFramework\Bootstrap\Coordinator; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; @@ -140,6 +142,8 @@ private function getGenerator(): Generator { ), $this->eventDispatcher, $this->container->get(LoggerInterface::class), + $this->container->get(PreviewMapper::class), + $this->container->get(StorageFactory::class), ); } return $this->generator; diff --git a/lib/public/IPreview.php b/lib/public/IPreview.php index 3c9eadd45774a..cbd0e0ae525e4 100644 --- a/lib/public/IPreview.php +++ b/lib/public/IPreview.php @@ -29,6 +29,11 @@ interface IPreview { */ public const MODE_COVER = 'cover'; + public const MIMETYPE_JPEG = 0; + public const MIMETYPE_WEBP = 1; + public const MIMETYPE_PNG = 2; + public const MIMETYPE_GIF = 3; + /** * In order to improve lazy loading a closure can be registered which will be * called in case preview providers are actually requested From 656e33e8daf2be3a51285715ea9a87bf462a6c94 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 21 Aug 2025 11:18:22 +0200 Subject: [PATCH 02/14] perf(preview): Add support for multibucket storage Signed-off-by: Carl Schwan --- lib/private/Preview/Db/Preview.php | 10 ++++++++-- lib/private/Preview/Generator.php | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index f2f0587108273..fbecd485421e9 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -17,6 +17,8 @@ /** * @method \int getFileId() * @method void setFileId(int $fileId) + * @method \int getStorageId() + * @method void setStorageId(\int $fileId) * @method \int getWidth() * @method void setWidth(int $width) * @method \int getHeight() @@ -41,6 +43,8 @@ class Preview extends Entity { protected ?int $fileId = null; + protected ?int $storageId = null; + protected ?int $width = null; protected ?int $height = null; @@ -56,10 +60,12 @@ class Preview extends Entity { protected ?bool $crop = null; protected ?string $etag = null; + protected ?int $version = null; public function __construct() { - $this->addType('fileId', Types::INTEGER); + $this->addType('fileId', Types::BIGINT); + $this->addType('storageId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); @@ -68,7 +74,7 @@ public function __construct() { $this->addType('isMax', Types::BOOLEAN); $this->addType('crop', Types::BOOLEAN); $this->addType('etag', Types::STRING); - $this->addType('version', Types::INTEGER); + $this->addType('version', Types::BIGINT); } public function getName(): string { diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 39e1de54e69d3..ed4e5d32177e5 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -552,6 +552,7 @@ private function getExtension($mimeType) { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId((int)$file->getStorage()->getId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); From 13c35c0f17d41a0c6e567c345021ee1c0ae9c4e6 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 21 Aug 2025 16:42:34 +0200 Subject: [PATCH 03/14] perf(preview): Migrate previews to the new optimized table Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 238 ++++++++++++++++++ .../Version33000Date20250819110529.php | 3 +- lib/composer/composer/autoload_classmap.php | 4 +- lib/composer/composer/autoload_static.php | 4 +- lib/private/BackgroundJob/JobList.php | 1 + .../ObjectStore/PrimaryObjectStoreConfig.php | 4 +- lib/private/Preview/Db/Preview.php | 4 +- lib/private/Preview/Generator.php | 25 +- .../Preview/Storage/IPreviewStorage.php | 8 + .../Preview/Storage/LocalPreviewStorage.php | 69 +++-- .../Storage/ObjectStorePreviewStorage.php | 26 +- lib/private/Preview/Storage/PreviewFile.php | 14 +- .../Preview/Storage/StorageFactory.php | 17 +- lib/private/PreviewManager.php | 1 - lib/private/Repair.php | 2 + lib/private/Repair/AddMovePreviewJob.php | 27 ++ lib/private/Setup.php | 2 + 17 files changed, 401 insertions(+), 48 deletions(-) create mode 100644 core/BackgroundJobs/MovePreviewJob.php create mode 100644 lib/private/Repair/AddMovePreviewJob.php diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php new file mode 100644 index 0000000000000..d75673e6d2d82 --- /dev/null +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -0,0 +1,238 @@ +appData = $appDataFactory->get('preview'); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + $this->setInterval(24 * 60 * 60); + } + + protected function run(mixed $argument): void { + try { + $this->doRun($argument); + } catch (\Throwable $exception) { + echo $exception->getMessage(); + throw $exception; + } + } + + private function doRun($argument): void { + if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { + //return; + } + + $emptyHierarchicalPreviewFolders = false; + + $startTime = time(); + while (true) { + $previewFolders = []; + + // Check new hierarchical preview folders first + if (!$emptyHierarchicalPreviewFolders) { + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) + ->setMaxResults(100); + + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = $pathSplit[count($pathSplit) - 2]; + $previewFolders[$fileId][] = $row['path']; + } + + if (!empty($previewFolders)) { + $this->processPreviews($previewFolders, false); + continue; + } + } + + // And then the flat preview folder (legacy) + $emptyHierarchicalPreviewFolders = true; + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.jpg'))) + ->setMaxResults(100); + + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = $pathSplit[count($pathSplit) - 2]; + array_pop($pathSplit); + $path = implode('/', $pathSplit); + if (!isset($previewFolders[$fileId])) { + $previewFolders[$fileId] = []; + } + if (!in_array($path, $previewFolders[$fileId])) { + $previewFolders[$fileId][] = $path; + } + } + + if (empty($previewFolders)) { + break; + } else { + $this->processPreviews($previewFolders, true); + } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 3600) { + return; + } + } + + // Delete any left over preview directory + $this->appData->getFolder('.')->delete(); + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + } + + /** + * @param array $previewFolders + */ + private function processPreviews(array $previewFolders, bool $simplePaths): void { + foreach ($previewFolders as $fileId => $previewFolder) { + $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + $folder = $this->appData->getFolder($internalPath); + + /** + * @var list $previewFiles + */ + $previewFiles = []; + + foreach ($folder->getDirectoryListing() as $previewFile) { + [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); + $nameSplit = explode('-', $baseName); + + // TODO VERSION/PREFIX extraction + + $width = $nameSplit[0]; + $height = $nameSplit[1]; + + if (isset($nameSplit[2])) { + $crop = $nameSplit[2] === 'crop'; + $max = $nameSplit[2] === 'max'; + } + + $previewFiles[] = [ + 'file' => $previewFile, + 'width' => $width, + 'height' => $height, + 'crop' => $crop, + 'max' => $max, + 'extension' => $extension, + 'size' => $previewFile->getSize(), + 'mtime' => $previewFile->getMTime(), + ]; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId))); + + $result = $qb->executeQuery(); + $result = $result->fetchAll(); + + if (count($result) > 0) { + foreach ($previewFiles as $previewFile) { + $preview = new Preview(); + $preview->setFileId((int)$fileId); + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setMtime($previewFile['mtime']); + $preview->setWidth($previewFile['width']); + $preview->setHeight($previewFile['height']); + $preview->setCrop($previewFile['crop']); + $preview->setIsMax($previewFile['max']); + $preview->setMimetype(match ($previewFile['extension']) { + 'png' => IPreview::MIMETYPE_PNG, + 'webp' => IPreview::MIMETYPE_WEBP, + 'gif' => IPreview::MIMETYPE_GIF, + default => IPreview::MIMETYPE_JPEG, + }); + $preview->setSize($previewFile['size']); + try { + $preview = $this->previewMapper->insert($preview); + } catch (Exception $e) { + // We already have this preview in the preview table, skip + continue; + } + + try { + $this->storageFactory->migratePreview($preview, $previewFile['file']); + $previewFile['file']->delete(); + } catch (Exception $e) { + $this->previewMapper->delete($preview); + throw $e; + } + + } + } + + $this->deleteFolder($internalPath, $folder); + } + } + + public static function getInternalFolder(string $name, bool $simplePaths): string { + if ($simplePaths) { + return '/' . $name; + } + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + private function deleteFolder(string $path, ISimpleFolder $folder): void { + $folder->delete(); + + $current = $path; + + while (true) { + $current = dirname($current); + if ($current === '/' || $current === '.' || $current === '') { + break; + } + + + $folder = $this->appData->getFolder($current); + if (count($folder->getDirectoryListing()) !== 0) { + break; + } + $folder->delete(); + } + } +} diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index fcc008b428381..496ee849beb36 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -30,6 +30,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); @@ -38,7 +39,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 80191ae4b7fef..d3eb6553b8cb9 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1253,6 +1253,7 @@ 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => $baseDir . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1528,7 +1529,7 @@ 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', - 'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1958,6 +1959,7 @@ 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => $baseDir . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 923cf1d5f0b09..d14e3989a6a12 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1294,6 +1294,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1569,7 +1570,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', - 'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1999,6 +2000,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index c00a51e385166..302bee22cd5cb 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -321,6 +321,7 @@ private function buildJob(array $row): ?IJob { /** @var IJob $job */ $job = \OCP\Server::get($row['class']); } catch (QueryException $e) { + echo $e->getMessage(); if (class_exists($row['class'])) { $class = $row['class']; $job = new $class(); diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 4a31d2e51f5c9..cd2983d385c9e 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix: ?string, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix?: string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( @@ -155,7 +155,7 @@ public function getObjectStoreConfigs(): ?array { } /** - * @param array|string $config + * @param array{multibucket?: bool, objectPrefix?: string, ...}|string $config * @return string|ObjectStoreConfig */ private function validateObjectStoreConfig(array|string $config): array|string { diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index fbecd485421e9..c88cebedc9155 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -78,7 +78,7 @@ public function __construct() { } public function getName(): string { - $path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->getCrop()) { $path .= '-crop'; } @@ -102,7 +102,7 @@ public function getMimetypeValue(): string { public function getExtension(): string { return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'jpeg', + IPreview::MIMETYPE_JPEG => 'jpg', IPreview::MIMETYPE_PNG => 'png', IPreview::MIMETYPE_WEBP => 'webp', IPreview::MIMETYPE_GIF => 'gif', diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index ed4e5d32177e5..29590d6fa93c3 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,13 +12,11 @@ use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\InMemoryFile; use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -27,7 +25,6 @@ use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; use Psr\Log\LoggerInterface; -use function Symfony\Component\Translation\t; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -36,7 +33,6 @@ class Generator { public function __construct( private IConfig $config, private IPreview $previewManager, - private IAppData $appData, private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, @@ -113,7 +109,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = null; + $previewVersion = -1; if ($file instanceof IVersionedPreviewFile) { $previewVersion = (int)$file->getPreviewVersion(); } @@ -135,8 +131,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim throw new NotFoundException('The maximum preview sizes are zero or less pixels'); } - $preview = null; - + $previewFile = null; foreach ($specifications as $specification) { $width = $specification['width'] ?? -1; $height = $specification['height'] ?? -1; @@ -155,7 +150,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $preview = $maxPreview; + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); continue; } @@ -181,9 +176,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); - // New file, augment our array - //$previews[] = $preview; + $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); @@ -307,7 +300,7 @@ public function getNumConcurrentPreviews(string $type): int { * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview { + private function getMaxPreview(array $previews, File $file, string $mimeType, int $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { @@ -322,7 +315,7 @@ private function getMaxPreview(array $previews, File $file, string $mimeType, ?i return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -368,8 +361,8 @@ private function generateProviderPreview(File $file, int $width, int $height, bo throw new NotFoundException('No provider successfully handled the preview generation'); } - private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string { - $path = ($version ? $version . '-' : '') . $width . '-' . $height; + private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): string { + $path = ($version !== -1 ? $version . '-' : '') . $width . '-' . $height; if ($crop) { $path .= '-crop'; } @@ -549,7 +542,7 @@ private function getExtension($mimeType) { * @throws NotPermittedException * @throws \OCP\DB\Exception */ - public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { + public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId((int)$file->getStorage()->getId()); diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index a989c492c7606..8ebf98b7493ff 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -2,6 +2,7 @@ namespace OC\Preview\Storage; +use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OCP\Files\NotPermittedException; @@ -19,4 +20,11 @@ public function writePreview(Preview $preview, $stream): false|int; public function readPreview(Preview $preview); public function deletePreview(Preview $preview); + + /** + * Migration helper + * + * To remove at some point + */ + public function migratePreview(Preview $preview, SimpleFile $file): void; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 8df75f317ff60..5971113cb53c0 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -1,43 +1,80 @@ rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); - ['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath); - $currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY; - mkdir($currentDir); - foreach (explode('/', $dirname) as $suffix) { - $currentDir .= "/$suffix"; - mkdir($currentDir); - } - $file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w"); + $this->createParentFiles($previewPath); + $file = @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'w'); return fwrite($file, $stream); } public function readPreview(Preview $preview) { $previewPath = $this->constructPath($preview); - return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r"); + return @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'r'); } public function deletePreview(Preview $preview) { $previewPath = $this->constructPath($preview); - @unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath); + @unlink($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath); } private function constructPath(Preview $preview): string { return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } + + private function createParentFiles($path) { + ['basename' => $basename, 'dirname' => $dirname] = pathinfo($path); + $currentDir = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY; + mkdir($currentDir); + foreach (explode('/', $dirname) as $suffix) { + $currentDir .= "/$suffix"; + mkdir($currentDir); + } + } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + $instanceId = $this->config->getSystemValueString('instanceid'); + $previewPath = $this->constructPath($preview); + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; + $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; + if (!file_exists($sourcePath)) { + // legacy flat directory + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + } + if (file_exists($destinationPath)) { + return; + } + $this->createParentFiles($previewPath); + echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; + $ok = copy($sourcePath, $destinationPath); + if (!$ok) { + throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index fc9a64ed761f6..11944dbf846aa 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -1,8 +1,17 @@ objectPrefix = $parameters['objectPrefix'] . 'preview:'; } @@ -47,4 +59,14 @@ public function deletePreview(Preview $preview) { private function constructUrn(Preview $preview): string { return $this->objectPrefix . $preview->getId(); } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + if (isset($this->parameters['objectPrefix'])) { + $objectPrefix = $this->parameters['objectPrefix']; + } else { + $objectPrefix = 'urn:oid:'; + } + + $this->objectStore->copyObject($objectPrefix . $file->getId(), $this->constructUrn($preview)); + } } diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index f9e4cc0f59aa0..c9381ce933cda 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -1,5 +1,13 @@ getBackend()->writePreview($preview, $stream); @@ -20,7 +24,7 @@ public function readPreview(Preview $preview) { return $this->getBackend()->readPreview($preview); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { $this->getBackend()->deletePreview($preview); } @@ -35,10 +39,13 @@ private function getBackend(): IPreviewStorage { $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); } else { - $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); - $this->backend = new LocalPreviewStorage($configDataDirectory); + $this->backend = new LocalPreviewStorage($this->config); } return $this->backend; } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + $this->getBackend()->migratePreview($preview, $file); + } } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 7c8e1b13eb99f..3bc63a55adb93 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -135,7 +135,6 @@ private function getGenerator(): Generator { $this->generator = new Generator( $this->config, $this, - $this->appData, new GeneratorHelper( $this->rootFolder, $this->config diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 583604515afe1..6a123cc56dd96 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -13,6 +13,7 @@ use OC\Repair\AddCleanupDeletedUsersBackgroundJob; use OC\Repair\AddCleanupUpdaterBackupsJob; use OC\Repair\AddMetadataGenerationJob; +use OC\Repair\AddMovePreviewJob; use OC\Repair\AddRemoveOldTasksBackgroundJob; use OC\Repair\CleanTags; use OC\Repair\CleanUpAbandonedApps; @@ -199,6 +200,7 @@ public static function getRepairSteps(): array { \OCP\Server::get(RemoveLegacyDatadirFile::class), \OCP\Server::get(AddCleanupDeletedUsersBackgroundJob::class), \OCP\Server::get(SanitizeAccountProperties::class), + \OCP\Server::get(AddMovePreviewJob::class), ]; } diff --git a/lib/private/Repair/AddMovePreviewJob.php b/lib/private/Repair/AddMovePreviewJob.php new file mode 100644 index 0000000000000..5edcd64d46e8e --- /dev/null +++ b/lib/private/Repair/AddMovePreviewJob.php @@ -0,0 +1,27 @@ +jobList->add(MovePreviewJob::class); + } +} diff --git a/lib/private/Setup.php b/lib/private/Setup.php index 15f0a4eb61700..8781127b30afc 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -15,6 +15,7 @@ use OC\Authentication\Token\PublicKeyTokenProvider; use OC\Authentication\Token\TokenCleanupJob; use OC\Core\BackgroundJobs\GenerateMetadataJob; +use OC\Core\BackgroundJobs\MovePreviewJob; use OC\Log\Rotate; use OC\Preview\BackgroundCleanupJob; use OC\TextProcessing\RemoveOldTasksBackgroundJob; @@ -505,6 +506,7 @@ public static function installBackgroundJobs(): void { $jobList->add(RemoveOldTasksBackgroundJob::class); $jobList->add(CleanupDeletedUsers::class); $jobList->add(GenerateMetadataJob::class); + $jobList->add(MovePreviewJob::class); } /** From 6008852232205a02df68ca4290c65c1f6b81ec17 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 28 Aug 2025 14:25:55 +0200 Subject: [PATCH 04/14] feat(preview): Support multibucket storage Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 2 +- lib/private/Preview/Generator.php | 6 +- lib/private/Preview/GeneratorHelper.php | 6 +- .../Preview/Storage/IPreviewStorage.php | 1 + .../Preview/Storage/LocalPreviewStorage.php | 2 +- .../Storage/ObjectStorePreviewStorage.php | 144 +++++++- .../Preview/Storage/StorageFactory.php | 7 +- tests/lib/Preview/GeneratorTest.php | 348 ++++++++---------- 8 files changed, 298 insertions(+), 218 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index d75673e6d2d82..d326fab88d3d6 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -197,7 +197,7 @@ private function processPreviews(array $previewFolders, bool $simplePaths): void try { $this->storageFactory->migratePreview($preview, $previewFile['file']); $previewFile['file']->delete(); - } catch (Exception $e) { + } catch (\Exception $e) { $this->previewMapper->delete($preview); throw $e; } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 29590d6fa93c3..e846a8221abad 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -174,6 +174,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim if ($maxPreviewImage === null) { $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } + assert($maxPreviewImage); $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); @@ -324,6 +325,7 @@ private function generateProviderPreview(File $file, int $width, int $height, bo } foreach ($providers as $providerClosure) { + $provider = $this->helper->getProvider($providerClosure); if (!($provider instanceof IProviderV2)) { continue; @@ -351,7 +353,7 @@ private function generateProviderPreview(File $file, int $width, int $height, bo } try { - return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version); + return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); } catch (NotPermittedException $e) { throw new NotFoundException(); } @@ -545,7 +547,7 @@ private function getExtension($mimeType) { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); - $previewEntry->setStorageId((int)$file->getStorage()->getId()); + $previewEntry->setStorageId((int)$file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); diff --git a/lib/private/Preview/GeneratorHelper.php b/lib/private/Preview/GeneratorHelper.php index e914dcc20020b..7114a412e36ea 100644 --- a/lib/private/Preview/GeneratorHelper.php +++ b/lib/private/Preview/GeneratorHelper.php @@ -45,11 +45,7 @@ public function getThumbnail(IProviderV2 $provider, File $file, $maxWidth, $maxH return $provider->getThumbnail($file, $maxWidth, $maxHeight) ?? false; } - /** - * @param ISimpleFile $maxPreview - * @return IImage - */ - public function getImage(ISimpleFile $maxPreview) { + public function getImage(ISimpleFile $maxPreview): IImage { $image = new OCPImage(); $image->loadFromData($maxPreview->getContent()); return $image; diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 8ebf98b7493ff..787518e53a40e 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -25,6 +25,7 @@ public function deletePreview(Preview $preview); * Migration helper * * To remove at some point + * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 5971113cb53c0..2c176948c79fd 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -72,7 +72,7 @@ public function migratePreview(Preview $preview, SimpleFile $file): void { } $this->createParentFiles($previewPath); echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; - $ok = copy($sourcePath, $destinationPath); + $ok = rename($sourcePath, $destinationPath); if (!$ok) { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 11944dbf846aa..a8fe79d4732db 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -11,23 +11,32 @@ namespace OC\Preview\Storage; use Icewind\Streams\CountWrapper; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +/** + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: array} + */ class ObjectStorePreviewStorage implements IPreviewStorage { - private string $objectPrefix = 'urn:oid:preview:'; /** - * @param array{objectPrefix?: string, ...} $parameters + * @var array<'root'|int, ObjectStoreDefinition> */ + private array $objectStoreCache = []; + + private bool $isMultibucketEnabled; + private bool $isMultibucketPreviewDistributionEnabled; + public function __construct( - private readonly IObjectStore $objectStore, - private readonly array $parameters, + private readonly PrimaryObjectStoreConfig $objectStoreConfig, + readonly private IConfig $config, ) { - if (isset($parameters['objectPrefix'])) { - $this->objectPrefix = $parameters['objectPrefix'] . 'preview:'; - } + $this->isMultibucketEnabled = is_array($config->getSystemValue('objectstore_multibucket')); + $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } public function writePreview(Preview $preview, $stream): false|int { @@ -44,29 +53,128 @@ public function writePreview(Preview $preview, $stream): false|int { $size = $writtenSize; }); - $this->objectStore->writeObject($this->constructUrn($preview), $countStream); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + + $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); return $size; } public function readPreview(Preview $preview) { - return $this->objectStore->readObject($this->constructUrn($preview)); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } public function deletePreview(Preview $preview) { - return $this->objectStore->deleteObject($this->constructUrn($preview)); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + return $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } - private function constructUrn(Preview $preview): string { - return $this->objectPrefix . $preview->getId(); + public function migratePreview(Preview $preview, SimpleFile $file): void { + foreach ([false, true] as $fallback) { + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + 'config' => $config, + ] = $this->getObjectStoreForPreview($preview, $fallback); + + $oldObjectPrefix = 'urn:oid:'; + if (isset($config['objectPrefix'])) { + $oldObjectPrefix = $config['objectPrefix']; + } + + try { + $store->copyObject($this->constructUrn($oldObjectPrefix, $file->getId()), $this->constructUrn($objectPrefix, $preview->getId())); + break; + } catch (NotFoundException $e) { + if (!$fallback && $this->isMultibucketPreviewDistributionEnabled) { + continue; + } + throw $e; + } + } } - public function migratePreview(Preview $preview, SimpleFile $file): void { - if (isset($this->parameters['objectPrefix'])) { - $objectPrefix = $this->parameters['objectPrefix']; - } else { - $objectPrefix = 'urn:oid:'; + /** + * @return ObjectStoreDefinition + */ + private function getMultiBucketObjectStore(int $number): array { + /** + * @var array{class: class-string, ...} $config + */ + $config = $this->config->getSystemValue('objectstore_multibucket'); + + if (!isset($config['arguments'])) { + $config['arguments'] = []; } - $this->objectStore->copyObject($objectPrefix . $file->getId(), $this->constructUrn($preview)); + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + $config['arguments']['bucket'] .= "-preview-$number"; + + $objectPrefix = 'urn:oid:preview:'; + if (isset($config['objectPrefix'])) { + $objectPrefix = $config['objectPrefix'] . 'preview:'; + } + + return [ + 'store' => new $config['class']($config['arguments']), + 'objectPrefix' => $objectPrefix, + 'config' => $config, + ]; + } + + /** + * @return ObjectStoreDefinition + */ + private function getRootObjectStore(): array { + if (!isset($this->objectStoreCache['root'])) { + $rootConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + $objectPrefix = 'urn:oid:preview:'; + if (isset($rootConfig['arguments']['objectPrefix'])) { + $objectPrefix = $rootConfig['arguments']['objectPrefix'] . 'preview:'; + } + $this->objectStoreCache['root'] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($rootConfig), + 'objectPrefix' => $objectPrefix, + ]; + } + return $this->objectStoreCache['root']; + } + + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { + if (!$this->isMultibucketEnabled || !$this->isMultibucketPreviewDistributionEnabled || $oldFallback) { + return $this->getRootObjectStore(); + } + + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + + if (!isset($this->objectStoreCache[$bucketNumber])) { + $this->objectStoreCache[$bucketNumber] = $this->getMultiBucketObjectStore($bucketNumber); + } + + return $this->objectStoreCache[$bucketNumber]; + } + + private function constructUrn(string $objectPrefix, int $id): string { + return $objectPrefix . $id; } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index a8ba57e8e4421..077395c8b93fb 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -33,11 +33,8 @@ private function getBackend(): IPreviewStorage { return $this->backend; } - $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - - if ($objectStoreConfig) { - $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); - $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); + if ($this->objectStoreConfig->hasObjectStore()) { + $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config); } else { $this->backend = new LocalPreviewStorage($this->config); } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index edf5418da6ef7..91fa353746814 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -7,11 +7,14 @@ namespace Test\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; +use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; use OCP\Files\SimpleFS\ISimpleFolder; @@ -20,110 +23,108 @@ use OCP\IPreview; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; +use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; - -class GeneratorTest extends \Test\TestCase { - /** @var IConfig&\PHPUnit\Framework\MockObject\MockObject */ - private $config; - - /** @var IPreview&\PHPUnit\Framework\MockObject\MockObject */ - private $previewManager; - - /** @var IAppData&\PHPUnit\Framework\MockObject\MockObject */ - private $appData; - - /** @var GeneratorHelper&\PHPUnit\Framework\MockObject\MockObject */ - private $helper; - - /** @var IEventDispatcher&\PHPUnit\Framework\MockObject\MockObject */ - private $eventDispatcher; - - /** @var Generator */ - private $generator; - - private LoggerInterface&\PHPUnit\Framework\MockObject\MockObject $logger; +use Test\TestCase; + +class GeneratorTest extends TestCase { + private IConfig&MockObject $config; + private IPreview&MockObject $previewManager; + private GeneratorHelper&MockObject $helper; + private IEventDispatcher&MockObject $eventDispatcher; + private Generator $generator; + private LoggerInterface&MockObject $logger; + private StorageFactory&MockObject $storageFactory; + private PreviewMapper&MockObject $previewMapper; protected function setUp(): void { parent::setUp(); $this->config = $this->createMock(IConfig::class); $this->previewManager = $this->createMock(IPreview::class); - $this->appData = $this->createMock(IAppData::class); $this->helper = $this->createMock(GeneratorHelper::class); $this->eventDispatcher = $this->createMock(IEventDispatcher::class); $this->logger = $this->createMock(LoggerInterface::class); + $this->previewMapper = $this->createMock(PreviewMapper::class); + $this->storageFactory = $this->createMock(StorageFactory::class); $this->generator = new Generator( $this->config, $this->previewManager, - $this->appData, $this->helper, $this->eventDispatcher, $this->logger, + $this->previewMapper, + $this->storageFactory ); } - public function testGetCachedPreview(): void { + private function getFile(int $fileId, string $mimeType): File { + $mountPoint = $this->createMock(IMountPoint::class); + $mountPoint->method('getNumericStorageId')->willReturn(42); $file = $this->createMock(File::class); $file->method('isReadable') ->willReturn(true); $file->method('getMimeType') - ->willReturn('myMimeType'); + ->willReturn($mimeType); $file->method('getId') - ->willReturn(42); + ->willReturn($fileId); + $file->method('getMountPoint') + ->willReturn($mountPoint); + return $file; + } + + public function testGetCachedPreview(): void { + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('1000-1000-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); - $previewFile->method('getName')->willReturn('256-256.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $previewFile]); + $maxPreview = new Preview(); + $maxPreview->setWidth(1000); + $maxPreview->setHeight(1000); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setCrop(false); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + + $previewFile = new Preview(); + $previewFile->setWidth(256); + $previewFile->setHeight(256); + $previewFile->setIsMax(false); + $previewFile->setSize(1000); + $previewFile->setVersion(-1); + $previewFile->setCrop(false); + $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame('256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } public function testGetNewPreview(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willThrowException(new NotFoundException()); - - $this->appData->method('newFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); $this->config->method('getSystemValue') ->willReturnCallback(function ($key, $default) { @@ -175,7 +176,7 @@ public function testGetNewPreview(): void { $image->method('dataMimeType')->willReturn('image/png'); $this->helper->method('getThumbnail') - ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image) { + ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image): false|IImage { if ($provider === $validProvider) { return $image; } else { @@ -186,29 +187,34 @@ public function testGetNewPreview(): void { $image->method('data') ->willReturn('my data'); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName')->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType')->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); - - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); - - $previewFolder->method('getDirectoryListing') - ->willReturn([]); - $previewFolder->method('newFile') - ->willReturnMap([ - ['2048-2048-max.png', 'my data', $maxPreview], - ['256-256.png', 'my resized data', $previewFile], - ]); - - $previewFolder->method('getFile') - ->with($this->equalTo('256-256.png')) - ->willThrowException(new NotFoundException()); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + + $this->previewMapper->method('insert') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->storageFactory->method('writePreview') + ->willReturnCallback(function (Preview $preview, string $data): int { + switch ($preview->getName()) { + case '2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case '256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + $this->fail("file name is wrong:". $preview->getName()); + }); $image = $this->getMockImage(2048, 2048, 'my resized data'); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); $this->eventDispatcher->expects($this->once()) @@ -216,39 +222,32 @@ public function testGetNewPreview(): void { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame('256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } public function testInvalidMimeType(): void { $this->expectException(NotFoundException::class); - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'invalidType'); $this->previewManager->method('isMimeSupported') ->with('invalidType') ->willReturn(false); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $previewFolder->method('getFile') - ->with($this->equalTo('1024-512-crop.png')) - ->willThrowException(new NotFoundException()); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -258,31 +257,31 @@ public function testInvalidMimeType(): void { } public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); - - - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $preview->method('getName')->willReturn('1024-512-crop.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $preview]); + $file = $this->getFile(42, 'myMimeType'); + + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + + $previewFile = new Preview(); + $previewFile->setWidth(1024); + $previewFile->setHeight(512); + $previewFile->setIsMax(false); + $previewFile->setSize(1000); + $previewFile->setCrop(true); + $previewFile->setVersion(-1); + $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->previewManager->expects($this->never()) ->method('isMimeSupported'); @@ -292,25 +291,15 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void ->with(new BeforePreviewFetchedEvent($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType')); $result = $this->generator->getPreview($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType'); - $this->assertSame($preview, $result); + $this->assertSame('1024-512-crop.png', $result->getName()); } public function testNoProvider(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $previewFolder->method('getDirectoryListing') - ->willReturn([]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); $this->previewManager->method('getProviders') ->willReturn([]); @@ -380,65 +369,52 @@ public static function dataSize(): array { ]; } - /** - * - * @param int $maxX - * @param int $maxY - * @param int $reqX - * @param int $reqY - * @param bool $crop - * @param string $mode - * @param int $expectedX - * @param int $expectedY - */ #[\PHPUnit\Framework\Attributes\DataProvider('dataSize')] - public function testCorrectSize($maxX, $maxY, $reqX, $reqY, $crop, $mode, $expectedX, $expectedY): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $crop, string $mode, int $expectedX, int $expectedY): void { + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $maxPreview = new Preview(); + $maxPreview->setWidth($maxX); + $maxPreview->setHeight($maxY); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn($maxX . '-' . $maxY . '-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); + $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimetypeValue(), 'image/png'); - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $filename = $expectedX . '-' . $expectedY; if ($crop) { $filename .= '-crop'; } $filename .= '.png'; - $previewFolder->method('getFile') - ->with($this->equalTo($filename)) - ->willThrowException(new NotFoundException()); $image = $this->getMockImage($maxX, $maxY); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $previewFolder->method('newFile') - ->with($this->equalTo($filename)) - ->willReturn($preview); + $this->previewMapper->method('insert') + ->willReturnCallback(function (Preview $preview) use ($filename): Preview { + $this->assertSame($preview->getName(), $filename); + return $preview; + }); + + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->storageFactory->method('writePreview') + ->willReturn(1000); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -446,9 +422,9 @@ public function testCorrectSize($maxX, $maxY, $reqX, $reqY, $crop, $mode, $expec $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview, $result); + $this->assertSame($maxPreview->getName(), $result->getName()); } else { - $this->assertSame($preview, $result); + $this->assertSame($filename, $result->getName()); } } From bba96678826ba9b6726484ff05a1d7fd48cfa8b7 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 29 Aug 2025 14:14:07 +0200 Subject: [PATCH 05/14] perf(preview): Adapt BackgroundCleanupJob to new previews table Signed-off-by: Carl Schwan --- build/psalm-baseline.xml | 18 -- lib/private/Preview/BackgroundCleanupJob.php | 185 +++++++----------- lib/private/Preview/Db/PreviewMapper.php | 30 +++ lib/private/Preview/Generator.php | 6 +- lib/public/AppFramework/Db/QBMapper.php | 1 - .../Preview/BeforePreviewFetchedEvent.php | 1 + .../lib/Preview/BackgroundCleanupJobTest.php | 122 ++---------- 7 files changed, 128 insertions(+), 235 deletions(-) diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 621e368f0a3f5..606cb3b5bb14b 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -3721,16 +3721,6 @@ - - - - - - - - - - @@ -3991,14 +3981,6 @@ - - - mode]]> - - - - - diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 3138abb1bf988..62e3303dc4e0f 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -8,23 +8,25 @@ */ namespace OC\Preview; -use OC\Preview\Storage\Root; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\Files\IMimeTypeLoader; -use OCP\Files\NotFoundException; -use OCP\Files\NotPermittedException; use OCP\IDBConnection; +/** + * @psalm-type FileId int + * @psalm-type StorageId int + */ class BackgroundCleanupJob extends TimedJob { public function __construct( ITimeFactory $timeFactory, - private IDBConnection $connection, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, - private bool $isCLI, + readonly private IDBConnection $connection, + readonly private PreviewMapper $previewMapper, + readonly private StorageFactory $storageFactory, + readonly private bool $isCLI, ) { parent::__construct($timeFactory); // Run at most once an hour @@ -32,88 +34,37 @@ public function __construct( $this->setTimeSensitivity(self::TIME_INSENSITIVE); } - public function run($argument) { - foreach ($this->getDeletedFiles() as $fileId) { - try { - $preview = $this->previewFolder->getFolder((string)$fileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue + public function run($argument): void { + foreach ($this->getDeletedFiles() as $chunk) { + foreach ($chunk as $storage => $fileIds) { + foreach ($this->previewMapper->getByFileIds($storage, $fileIds) as $previews) { + $previewIds = []; + foreach ($previews as $preview) { + $previewIds[] = $preview->getId(); + $this->storageFactory->deletePreview($preview); + } + + $this->previewMapper->deleteByIds($storage, $previewIds); + }; } } } + /** + * @return \Iterator> + */ private function getDeletedFiles(): \Iterator { - yield from $this->getOldPreviewLocations(); - yield from $this->getNewPreviewLocations(); - } - - private function getOldPreviewLocations(): \Iterator { - if ($this->connection->getShardDefinition('filecache')) { - // sharding is new enough that we don't need to support this - return; - } - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) - ->where( - $qb->expr()->andX( - $qb->expr()->isNull('b.fileid'), - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())), - $qb->expr()->like('a.name', $qb->createNamedParameter('__%')), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); - - if (!$this->isCLI) { - $qb->setMaxResults(10); - } - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - yield $row['name']; - } - - $cursor->closeCursor(); - } - - private function getNewPreviewLocations(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') - ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - if ($this->connection->getShardDefinition('filecache')) { - $chunks = $this->getAllPreviewIds($data['path'], 1000); + $chunks = $this->getAllPreviewIds(1000); foreach ($chunks as $chunk) { - yield from $this->findMissingSources($chunk); + foreach ($chunk as $storage => $preview) { + yield [$storage => $this->findMissingSources($storage, $preview)]; + } } return; } - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - /* * Deleting a file will not delete related previews right away. * @@ -130,19 +81,12 @@ private function getNewPreviewLocations(): \Iterator { * If the related file is deleted, b.fileid will be null and the preview folder can be deleted. */ $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + $qb->select('p.storage_id', 'p.file_id') + ->from('previews', 'p') + ->leftJoin('p', 'filecache', 'f', $qb->expr()->eq( + 'p.file_id', 'f.fileid' )) - ->where( - $qb->expr()->andX( - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->isNull('b.fileid'), - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); + ->where($qb->expr()->isNull('f.fileid')); if (!$this->isCLI) { $qb->setMaxResults(10); @@ -150,29 +94,38 @@ private function getNewPreviewLocations(): \Iterator { $cursor = $qb->executeQuery(); + $lastStorageId = null; + /** @var FileId[] $tmpResult */ + $tmpResult = []; while ($row = $cursor->fetch()) { - yield $row['name']; + if ($lastStorageId === null) { + $lastStorageId = $row['storage_id']; + } else if ($lastStorageId !== $row['storage_id']) { + yield [$lastStorageId => $tmpResult]; + $tmpResult = []; + $lastStorageId = $row['storage_id']; + } + $tmpResult[] = $row['file_id']; + } + + if (!empty($tmpResult)) { + yield [$lastStorageId => $tmpResult]; } $cursor->closeCursor(); } - private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterator { - // See `getNewPreviewLocations` for some more info about the logic here - $like = $this->connection->escapeLikeParameter($previewRoot) . '/_/_/_/_/_/_/_/%'; - + /** + * @return \Iterator> + */ + private function getAllPreviewIds(int $chunkSize): \Iterator { $qb = $this->connection->getQueryBuilder(); - $qb->select('name', 'fileid') - ->from('filecache') + $qb->select('id', 'file_id', 'storage_id') + ->from('previews') ->where( - $qb->expr()->andX( - $qb->expr()->eq('storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->like('path', $qb->createNamedParameter($like)), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->gt('fileid', $qb->createParameter('min_id')), - ) + $qb->expr()->gt('id', $qb->createParameter('min_id')), ) - ->orderBy('fileid', 'ASC') + ->orderBy('id', 'ASC') ->setMaxResults($chunkSize); $minId = 0; @@ -180,21 +133,33 @@ private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterato $qb->setParameter('min_id', $minId); $rows = $qb->executeQuery()->fetchAll(); if (count($rows) > 0) { - $minId = $rows[count($rows) - 1]['fileid']; - yield array_map(function ($row) { - return (int)$row['name']; - }, $rows); + $minId = $rows[count($rows) - 1]['id']; + $result = []; + foreach ($rows as $row) { + if (!isset($result[$row['storage_id']])) { + $result[$row['storage_id']] = []; + } + $result[$row['storage_id']][] = $row['file_id']; + } + yield $result; } else { break; } } } - private function findMissingSources(array $ids): array { + /** + * @param FileId[] $ids + * @return FileId[] + */ + private function findMissingSources(int $storage, array $ids): array { $qb = $this->connection->getQueryBuilder(); $qb->select('fileid') ->from('filecache') - ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + ->where($qb->expr()->andX( + $qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)), + $qb->expr()->eq('storage', $qb->createNamedParameter($storage, IQueryBuilder::PARAM_INT)), + )); $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); return array_diff($ids, $found); } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 5faa508c8e03b..6202363815050 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -63,4 +63,34 @@ public function getPreview(int $fileId, int $width, int $height, string $mode, i return null; } } + + /** + * @param int[] $fileIds + * @return array + */ + public function getByFileIds(int $storageId, array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where($selectQb->expr()->andX( + $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + )); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + /** + * @param int[] $previewIds + */ + public function deleteByIds(int $storageId, array $previewIds): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::TABLE_NAME) + ->where($qb->expr()->andX( + $qb->expr()->eq('storage_id', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)), + $qb->expr()->in('id', $qb->createNamedParameter($previewIds, IQueryBuilder::PARAM_INT_ARRAY)) + ))->executeStatement(); + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index e846a8221abad..2dfbfa6db62d1 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -174,7 +174,6 @@ public function generatePreviews(File $file, array $specifications, ?string $mim if ($maxPreviewImage === null) { $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } - assert($maxPreviewImage); $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); @@ -508,7 +507,6 @@ private function generatePreview( self::unguardWithSemaphore($sem); } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); if ($cacheResult) { $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); @@ -519,11 +517,9 @@ private function generatePreview( } /** - * @param string $mimeType - * @return null|string * @throws \InvalidArgumentException */ - private function getExtension($mimeType) { + private function getExtension(string $mimeType): string { switch ($mimeType) { case 'image/png': return 'png'; diff --git a/lib/public/AppFramework/Db/QBMapper.php b/lib/public/AppFramework/Db/QBMapper.php index 7fb5b2a9afd8d..d80bb5aec8b97 100644 --- a/lib/public/AppFramework/Db/QBMapper.php +++ b/lib/public/AppFramework/Db/QBMapper.php @@ -84,7 +84,6 @@ public function delete(Entity $entity): Entity { return $entity; } - /** * Creates a new entry in the db from an entity * diff --git a/lib/public/Preview/BeforePreviewFetchedEvent.php b/lib/public/Preview/BeforePreviewFetchedEvent.php index 8ab875070d99c..69cd281ac0260 100644 --- a/lib/public/Preview/BeforePreviewFetchedEvent.php +++ b/lib/public/Preview/BeforePreviewFetchedEvent.php @@ -21,6 +21,7 @@ */ class BeforePreviewFetchedEvent extends \OCP\EventDispatcher\Event { /** + * @param null|IPreview::MODE_FILL|IPreview::MODE_COVER $mode * @since 25.0.1 */ public function __construct( diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index ab904f2b49933..89503085fa464 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -9,7 +9,10 @@ use OC\Files\Storage\Temporary; use OC\Preview\BackgroundCleanupJob; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Storage\Root; +use OC\Preview\Storage\StorageFactory; use OC\PreviewManager; use OC\SystemConfig; use OCP\App\IAppManager; @@ -42,6 +45,8 @@ class BackgroundCleanupJobTest extends \Test\TestCase { private IRootFolder $rootFolder; private IMimeTypeLoader $mimeTypeLoader; private ITimeFactory $timeFactory; + private PreviewMapper $previewMapper; + private StorageFactory $previewStorageFactory; protected function setUp(): void { parent::setUp(); @@ -65,6 +70,8 @@ protected function setUp(): void { $this->rootFolder = Server::get(IRootFolder::class); $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); $this->timeFactory = Server::get(ITimeFactory::class); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->previewStorageFactory = Server::get(StorageFactory::class); } protected function tearDown(): void { @@ -78,13 +85,6 @@ protected function tearDown(): void { parent::tearDown(); } - private function getRoot(): Root { - return new Root( - Server::get(IRootFolder::class), - Server::get(SystemConfig::class) - ); - } - private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); @@ -99,130 +99,50 @@ private function setup11Previews(): array { return $files; } - private function countPreviews(Root $previewRoot, array $fileIds): int { - $i = 0; - - foreach ($fileIds as $fileId) { - try { - $previewRoot->getFolder((string)$fileId); - } catch (NotFoundException $e) { - continue; - } - - $i++; - } - - return $i; + private function countPreviews(PreviewMapper $previewMapper, array $fileIds): int { + $previews = $previewMapper->getAvailablePreviews($fileIds); + return array_reduce($previews, fn (int $result, array $previews) => $result + count($previews), 0); } public function testCleanupSystemCron(): void { $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); - - $root = $this->getRoot(); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, true); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); } public function testCleanupAjax(): void { if ($this->connection->getShardDefinition('filecache')) { $this->markTestSkipped('ajax cron is not supported for sharded setups'); - return; } $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); - - $root = $this->getRoot(); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, false); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job->run([]); - - $root = $this->getRoot(); - $this->assertSame(1, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); - } - - public function testOldPreviews(): void { - if ($this->connection->getShardDefinition('filecache')) { - $this->markTestSkipped('old previews are not supported for sharded setups'); - return; - } - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - $f1 = $appdata->newFolder('123456781'); - $f1->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder('123456782'); - $f2->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder((string)PHP_INT_MAX - 1); - $f2->newFile('foo.jpg', 'foo'); - - /* - * Cleanup of OldPreviewLocations should only remove numeric folders on AppData level, - * therefore these files should stay untouched. - */ - $appdata->getFolder('/')->newFile('not-a-directory', 'foo'); - $appdata->getFolder('/')->newFile('133742', 'bar'); - - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - // AppData::getDirectoryListing filters all non-folders - $this->assertSame(3, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } - - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->getRoot(), $this->mimeTypeLoader, true); + $this->assertSame(1, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - // Check if the files created above are still present - // Remember: AppData::getDirectoryListing filters all non-folders - $this->assertSame(0, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } + $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); } } From b0357663b96cf4a3a9bd34194466a4ce76e4f41d Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Wed, 10 Sep 2025 10:51:41 +0200 Subject: [PATCH 06/14] perf(preview): Optimize migration and simplify DB layout * Simplify migration by not moving the actual files and just updating the DB * Don't store the storageid in the preview table as it is not needed * Start adding tests Signed-off-by: Carl Schwan --- apps/files/lib/Command/ScanAppData.php | 5 + core/BackgroundJobs/MovePreviewJob.php | 6 +- .../Version33000Date20250819110529.php | 23 ++- lib/composer/composer/LICENSE | 2 + lib/composer/composer/autoload_classmap.php | 1 - lib/composer/composer/autoload_static.php | 13 +- .../ObjectStorePreviewCacheMountProvider.php | 138 ----------------- .../ObjectStore/PrimaryObjectStoreConfig.php | 6 + lib/private/Preview/Db/Preview.php | 25 ++- lib/private/Preview/Db/PreviewMapper.php | 43 ++++-- lib/private/Preview/Generator.php | 17 +-- .../Preview/Storage/LocalPreviewStorage.php | 22 ++- .../Storage/ObjectStorePreviewStorage.php | 144 +++++++----------- .../Preview/Storage/StorageFactory.php | 4 +- lib/private/Server.php | 2 - ...jectStorePreviewCacheMountProviderTest.php | 95 ------------ tests/lib/Files/Storage/Storage.php | 1 + tests/lib/Preview/MovePreviewJobTest.php | 32 ++++ tests/lib/Preview/PreviewMapperTest.php | 79 ++++++++++ 19 files changed, 291 insertions(+), 367 deletions(-) delete mode 100644 lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php delete mode 100644 tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php create mode 100644 tests/lib/Preview/MovePreviewJobTest.php create mode 100644 tests/lib/Preview/PreviewMapperTest.php diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 0e08c6a8cfe41..385e0624b3a92 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -51,6 +51,11 @@ protected function configure(): void { } protected function scanFiles(OutputInterface $output, string $folder): int { + if ($folder === 'preview') { + $output->writeln('Scanning the preview folder is not supported.'); + return self::FAILURE; + } + try { /** @var Folder $appData */ $appData = $this->getAppDataFolder(); diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index d326fab88d3d6..3541a128a46fc 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -8,6 +8,7 @@ namespace OC\Core\BackgroundJobs; +use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OC\Preview\Storage\StorageFactory; @@ -131,11 +132,12 @@ private function processPreviews(array $previewFolders, bool $simplePaths): void $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { + /** @var SimpleFile $previewFile */ [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); $nameSplit = explode('-', $baseName); @@ -173,7 +175,7 @@ private function processPreviews(array $previewFolders, bool $simplePaths): void foreach ($previewFiles as $previewFile) { $preview = new Preview(); $preview->setFileId((int)$fileId); - $preview->setStorageId($result[0]['storage']); + $preview->setOldFileId($previewFile['file']->getId()); $preview->setEtag($result[0]['etag']); $preview->setMtime($previewFile['mtime']); $preview->setWidth($previewFile['width']); diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 496ee849beb36..5e607024ec730 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -11,12 +11,15 @@ use Closure; use OCP\DB\ISchemaWrapper; use OCP\DB\Types; +use OCP\Migration\Attributes\CreateTable; use OCP\Migration\IOutput; use OCP\Migration\SimpleMigrationStep; /** * */ +#[CreateTable(table: 'preview', description: 'Holds the preview data')] +#[CreateTable(table: 'preview_locations', description: 'Holds the preview location in an object store')] class Version33000Date20250819110529 extends SimpleMigrationStep { /** @@ -26,21 +29,33 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); + if (!$schema->hasTable('preview_locations')) { + $table = $schema->createTable('preview_locations'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('bucket_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('object_store_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->setPrimaryKey(['id']); + } + if (!$schema->hasTable('previews')) { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); - $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); + $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); - $table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); - $table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]); - $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('source_mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); + $table->addIndex(['file_id']); $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); } diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index 62ecfd8d0046b..f27399a042d95 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,3 +1,4 @@ + Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -17,3 +18,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d3eb6553b8cb9..5c168a25de3b6 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1687,7 +1687,6 @@ 'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => $baseDir . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d14e3989a6a12..423b4cc6b575f 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 ); public static $prefixLengthsPsr4 = array ( - 'O' => + 'O' => array ( 'OC\\Core\\' => 8, 'OC\\' => 3, 'OCP\\' => 4, ), - 'N' => + 'N' => array ( 'NCU\\' => 4, ), ); public static $prefixDirsPsr4 = array ( - 'OC\\Core\\' => + 'OC\\Core\\' => array ( 0 => __DIR__ . '/../../..' . '/core', ), - 'OC\\' => + 'OC\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/private', ), - 'OCP\\' => + 'OCP\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/public', ), - 'NCU\\' => + 'NCU\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/unstable', ), @@ -1728,7 +1728,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php', diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php deleted file mode 100644 index 1546ef98f5020..0000000000000 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ /dev/null @@ -1,138 +0,0 @@ -logger = $logger; - $this->config = $config; - } - - /** - * @return MountPoint[] - * @throws \Exception - */ - public function getRootMounts(IStorageFactory $loader): array { - if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { - return []; - } - if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { - return []; - } - - $instanceId = $this->config->getSystemValueString('instanceid', ''); - $mountPoints = []; - $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; - $i = 0; - foreach ($directoryRange as $parent) { - foreach ($directoryRange as $child) { - $mountPoints[] = new MountPoint( - AppdataPreviewObjectStoreStorage::class, - '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, - $this->getMultiBucketObjectStore($i), - $loader, - null, - null, - self::class - ); - $i++; - } - } - - $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); - $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); - $fakeRootStorageJail = new Jail([ - 'storage' => $fakeRootStorage, - 'root' => '/appdata_' . $instanceId . '/preview', - ]); - - // add a fallback location to be able to fetch existing previews from the old bucket - $mountPoints[] = new MountPoint( - $fakeRootStorageJail, - '/appdata_' . $instanceId . '/preview/old-multibucket', - null, - $loader, - null, - null, - self::class - ); - - return $mountPoints; - } - - protected function getMultiBucketObjectStore(int $number): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - - $config['arguments']['bucket'] .= "-preview-$number"; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['internal-id'] = $number; - - return $config['arguments']; - } - - protected function getMultiBucketObjectStoreForRoot(): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - $config['arguments']['bucket'] .= '0'; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - return $config['arguments']; - } -} diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index cd2983d385c9e..02bc28f376e74 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -119,12 +119,14 @@ public function getObjectStoreConfigs(): ?array { 'default' => 'server1', 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), 'root' => 'server1', + 'preview' => 'server1', ]; } elseif ($objectStore) { if (!isset($objectStore['default'])) { $objectStore = [ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => $objectStore, ]; } @@ -132,6 +134,10 @@ public function getObjectStoreConfigs(): ?array { $objectStore['root'] = 'default'; } + if (!isset($objectStore['preview'])) { + $objectStore['preview'] = 'default'; + } + if (!is_string($objectStore['default'])) { throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index c88cebedc9155..b4ebe8b4805e5 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -17,8 +17,12 @@ /** * @method \int getFileId() * @method void setFileId(int $fileId) - * @method \int getStorageId() - * @method void setStorageId(\int $fileId) + * @method \int getOldFileId() // Old location in the file-cache table, for legacy compatibility + * @method void setOldFileId(int $fileId) + * @method \int getLocationId() + * @method void setLocationId(int $locationId) + * @method \string getBucketName() + * @method \string getObjectStoreName() * @method \int getWidth() * @method void setWidth(int $width) * @method \int getHeight() @@ -43,7 +47,11 @@ class Preview extends Entity { protected ?int $fileId = null; - protected ?int $storageId = null; + protected ?int $oldFileId = null; + + protected ?int $locationId = null; + protected ?string $bucketName = null; + protected ?string $objectStoreName = null; protected ?int $width = null; @@ -65,7 +73,8 @@ class Preview extends Entity { public function __construct() { $this->addType('fileId', Types::BIGINT); - $this->addType('storageId', Types::BIGINT); + $this->addType('oldFileId', Types::BIGINT); + $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); @@ -108,4 +117,12 @@ public function getExtension(): string { IPreview::MIMETYPE_GIF => 'gif', }; } + + public function setBucketName(string $bucketName): void { + $this->bucketName = $bucketName; + } + + public function setObjectStoreName(string $objectStoreName): void { + $this->objectStoreName = $objectStoreName; + } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 6202363815050..e0ec05ef88f93 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -22,6 +22,7 @@ class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; + private const LOCATION_TABLE_NAME = 'preview_locations'; public function __construct(IDBConnection $db) { parent::__construct($db, self::TABLE_NAME, Preview::class); @@ -34,8 +35,7 @@ public function __construct(IDBConnection $db) { */ public function getAvailablePreviews(array $fileIds): array { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where( $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), ); @@ -48,8 +48,7 @@ public function getAvailablePreviews(array $fileIds): array { public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where( $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), @@ -68,10 +67,9 @@ public function getPreview(int $fileId, int $width, int $height, string $mode, i * @param int[] $fileIds * @return array */ - public function getByFileIds(int $storageId, array $fileIds): array { + public function getByFileIds(array $fileIds): array { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where($selectQb->expr()->andX( $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), )); @@ -85,12 +83,39 @@ public function getByFileIds(int $storageId, array $fileIds): array { /** * @param int[] $previewIds */ - public function deleteByIds(int $storageId, array $previewIds): void { + public function deleteByIds(array $previewIds): void { $qb = $this->db->getQueryBuilder(); $qb->delete(self::TABLE_NAME) ->where($qb->expr()->andX( - $qb->expr()->eq('storage_id', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)), $qb->expr()->in('id', $qb->createNamedParameter($previewIds, IQueryBuilder::PARAM_INT_ARRAY)) ))->executeStatement(); } + + protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { + return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') + ->from(self::TABLE_NAME, 'p') + ->join('p', 'preview_locations', 'l', $qb->expr()->eq( + 'p.location_id', 'l.id' + )); + } + + public function getLocationId(string $bucket, string $objectStore): int { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('id') + ->from(self::LOCATION_TABLE_NAME) + ->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket))) + ->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore))) + ->executeQuery(); + $data = $result->fetchOne(); + if ($data) { + return $data; + } else { + $qb->insert(self::LOCATION_TABLE_NAME) + ->values([ + 'bucket_name' => $qb->createNamedParameter($bucket), + 'object_store_name' => $qb->createNamedParameter($objectStore), + ])->executeStatement(); + return $qb->getLastInsertId(); + } + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 2dfbfa6db62d1..44a12624c3e1a 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -156,17 +156,13 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // Try to get a cached preview. Else generate (and store) one try { - /** @var ISimpleFile $previewFile */ - $previewFile = null; - // TODO(php8.4) replace by array_find - foreach ($previews as $p) { - if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) { - $previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper); - break; - } - } + $preview = array_find($previews, fn (Preview $preview): bool => $preview->getWidth() === $width + && $preview->getHeight() === $height && $preview->getMimetype() === $maxPreview->getMimetype() + && $preview->getVersion() === $previewVersion && $preview->getCrop() === $crop); - if ($previewFile === null) { + if ($preview) { + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); + } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } @@ -543,7 +539,6 @@ private function getExtension(string $mimeType): string { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); - $previewEntry->setStorageId((int)$file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 2c176948c79fd..d4d7e37c0e5d8 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -18,30 +18,35 @@ class LocalPreviewStorage implements IPreviewStorage { private const PREVIEW_DIRECTORY = '__preview'; - private readonly string $rootFolder; + private readonly string $instanceId; public function __construct( private readonly IConfig $config, ) { + $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); $this->createParentFiles($previewPath); - $file = @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'w'); + $file = @fopen($this->getPreviewRootFolder() . $previewPath, 'w'); return fwrite($file, $stream); } public function readPreview(Preview $preview) { $previewPath = $this->constructPath($preview); - return @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'r'); + return @fopen($this->getPreviewRootFolder() . $previewPath, 'r'); } public function deletePreview(Preview $preview) { $previewPath = $this->constructPath($preview); - @unlink($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath); + @unlink($this->getPreviewRootFolder() . $previewPath); + } + + public function getPreviewRootFolder(): string { + return $this->rootFolder . '/appdata_' . $this->instanceId . '/preview/'; } private function constructPath(Preview $preview): string { @@ -63,11 +68,14 @@ public function migratePreview(Preview $preview, SimpleFile $file): void { $previewPath = $this->constructPath($preview); $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; - if (!file_exists($sourcePath)) { - // legacy flat directory - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + if (file_exists($sourcePath)) { + return; // No need to migrate } + + // legacy flat directory + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); if (file_exists($destinationPath)) { + @unlink($sourcePath); // We already have a new preview, just delete the old one return; } $this->createParentFiles($previewPath); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index a8fe79d4732db..97313fd306f1d 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -14,28 +14,29 @@ use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; /** - * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: array} + * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: ObjectStoreConfig} */ class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array<'root'|int, ObjectStoreDefinition> + * @var array> */ private array $objectStoreCache = []; - private bool $isMultibucketEnabled; private bool $isMultibucketPreviewDistributionEnabled; public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, - readonly private IConfig $config, + IConfig $config, + readonly private PreviewMapper $previewMapper, ) { - $this->isMultibucketEnabled = is_array($config->getSystemValue('objectstore_multibucket')); $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } @@ -56,6 +57,7 @@ public function writePreview(Preview $preview, $stream): false|int { [ 'objectPrefix' => $objectPrefix, 'store' => $store, + 'config' => $config, ] = $this->getObjectStoreForPreview($preview); $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); @@ -79,102 +81,72 @@ public function deletePreview(Preview $preview) { } public function migratePreview(Preview $preview, SimpleFile $file): void { - foreach ([false, true] as $fallback) { - [ - 'objectPrefix' => $objectPrefix, - 'store' => $store, - 'config' => $config, - ] = $this->getObjectStoreForPreview($preview, $fallback); - - $oldObjectPrefix = 'urn:oid:'; - if (isset($config['objectPrefix'])) { - $oldObjectPrefix = $config['objectPrefix']; - } - - try { - $store->copyObject($this->constructUrn($oldObjectPrefix, $file->getId()), $this->constructUrn($objectPrefix, $preview->getId())); - break; - } catch (NotFoundException $e) { - if (!$fallback && $this->isMultibucketPreviewDistributionEnabled) { - continue; - } - throw $e; - } - } + // Just set the Preview::bucket and Preview::objectStore + $this->getObjectStoreForPreview($preview, true); } /** * @return ObjectStoreDefinition */ - private function getMultiBucketObjectStore(int $number): array { - /** - * @var array{class: class-string, ...} $config - */ - $config = $this->config->getSystemValue('objectstore_multibucket'); - - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - - $config['arguments']['bucket'] .= "-preview-$number"; - - $objectPrefix = 'urn:oid:preview:'; - if (isset($config['objectPrefix'])) { - $objectPrefix = $config['objectPrefix'] . 'preview:'; + private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { + if ($preview->getObjectStoreName() === null) { + $config = $this->objectStoreConfig->getObjectStoreConfiguration($oldFallback ? 'root' : 'preview'); + $objectStoreName = $this->objectStoreConfig->resolveAlias($oldFallback ? 'root' : 'preview'); + + $bucketName = $config['arguments']['bucket']; + if ($config['arguments']['multibucket']) { + if ($this->isMultibucketPreviewDistributionEnabled) { + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $bucketName .= '-preview-' . $bucketNumber; + } else { + $bucketName .= '0'; + } + } + $config['arguments']['bucket'] = $bucketName; + + $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); + $preview->setLocationId($locationId); + $preview->setObjectStoreName($objectStoreName); + $preview->setBucketName($bucketName); + } else { + $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); + $config['arguments']['bucket'] = $bucketName = $preview->getBucketName(); + $objectStoreName = $preview->getObjectStoreName(); } - return [ - 'store' => new $config['class']($config['arguments']), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; - } + $objectPrefix = $this->getObjectPrefix($preview, $config); - /** - * @return ObjectStoreDefinition - */ - private function getRootObjectStore(): array { - if (!isset($this->objectStoreCache['root'])) { - $rootConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - $objectPrefix = 'urn:oid:preview:'; - if (isset($rootConfig['arguments']['objectPrefix'])) { - $objectPrefix = $rootConfig['arguments']['objectPrefix'] . 'preview:'; - } - $this->objectStoreCache['root'] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($rootConfig), + if (!isset($this->objectStoreCache[$objectStoreName])) { + $this->objectStoreCache[$objectStoreName] = []; + $this->objectStoreCache[$objectStoreName][$bucketName] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($config), 'objectPrefix' => $objectPrefix, + 'config' => $config, + ]; + } elseif (!isset($this->objectStoreCache[$objectStoreName][$bucketName])) { + $this->objectStoreCache[$objectStoreName][$bucketName] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($config), + 'objectPrefix' => $objectPrefix, + 'config' => $config, ]; - } - return $this->objectStoreCache['root']; - } - - /** - * @return ObjectStoreDefinition - */ - private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { - if (!$this->isMultibucketEnabled || !$this->isMultibucketPreviewDistributionEnabled || $oldFallback) { - return $this->getRootObjectStore(); - } - - $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); - $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); - - if (!isset($this->objectStoreCache[$bucketNumber])) { - $this->objectStoreCache[$bucketNumber] = $this->getMultiBucketObjectStore($bucketNumber); } - return $this->objectStoreCache[$bucketNumber]; + return $this->objectStoreCache[$objectStoreName][$bucketName]; } private function constructUrn(string $objectPrefix, int $id): string { return $objectPrefix . $id; } + + public function getObjectPrefix(Preview $preview, array $config): string { + if ($preview->getOldFileId()) { + return $config['arguments']['objectPrefix'] ?? 'uri:oid:'; + } + if (isset($config['arguments']['objectPrefix'])) { + return $config['arguments']['objectPrefix'] . 'preview:'; + } else { + return 'uri:oid:preview:'; + } + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 077395c8b93fb..0755bacd0b993 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -5,6 +5,7 @@ use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OCP\IConfig; class StorageFactory implements IPreviewStorage { @@ -13,6 +14,7 @@ class StorageFactory implements IPreviewStorage { public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, ) { } @@ -34,7 +36,7 @@ private function getBackend(): IPreviewStorage { } if ($this->objectStoreConfig->hasObjectStore()) { - $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config); + $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { $this->backend = new LocalPreviewStorage($this->config); } diff --git a/lib/private/Server.php b/lib/private/Server.php index 4445788ec4e24..fa88d8353e3b2 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -49,7 +49,6 @@ use OC\Files\Mount\CacheMountProvider; use OC\Files\Mount\LocalHomeMountProvider; use OC\Files\Mount\ObjectHomeMountProvider; -use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; use OC\Files\Mount\RootMountProvider; use OC\Files\Node\HookConnector; use OC\Files\Node\LazyRoot; @@ -789,7 +788,6 @@ public function __construct($webRoot, \OC\Config $config) { $manager->registerHomeProvider(new LocalHomeMountProvider()); $manager->registerHomeProvider(new ObjectHomeMountProvider($objectStoreConfig)); $manager->registerRootProvider(new RootMountProvider($objectStoreConfig, $config)); - $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config)); return $manager; }); diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php deleted file mode 100644 index 9060bf0d5f56d..0000000000000 --- a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php +++ /dev/null @@ -1,95 +0,0 @@ -logger = $this->createMock(LoggerInterface::class); - $this->config = $this->createMock(IConfig::class); - $this->loader = $this->createMock(StorageFactory::class); - - $this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config); - } - - public function testNoMultibucketObjectStorage(): void { - $this->config->expects($this->once()) - ->method('getSystemValue') - ->with('objectstore_multibucket') - ->willReturn(null); - - $this->assertEquals([], $this->provider->getRootMounts($this->loader)); - } - - public function testMultibucketObjectStorage(): void { - $objectstoreConfig = [ - 'class' => S3::class, - 'arguments' => [ - 'bucket' => 'abc', - 'num_buckets' => 64, - 'key' => 'KEY', - 'secret' => 'SECRET', - 'hostname' => 'IP', - 'port' => 'PORT', - 'use_ssl' => false, - 'use_path_style' => true, - ], - ]; - $this->config->expects($this->any()) - ->method('getSystemValue') - ->willReturnCallback(function ($config) use ($objectstoreConfig) { - if ($config === 'objectstore_multibucket') { - return $objectstoreConfig; - } elseif ($config === 'objectstore.multibucket.preview-distribution') { - return true; - } - return null; - }); - $this->config->expects($this->once()) - ->method('getSystemValueString') - ->with('instanceid') - ->willReturn('INSTANCEID'); - - $mounts = $this->provider->getRootMounts($this->loader); - - // 256 mounts for the subfolders and 1 for the fake root - $this->assertCount(257, $mounts); - - // do some sanity checks if they have correct mount point paths - $this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint()); - $this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint()); - // also test the path of the fake bucket - $this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint()); - } -} diff --git a/tests/lib/Files/Storage/Storage.php b/tests/lib/Files/Storage/Storage.php index 8110ab6e8cc93..60e696739eed5 100644 --- a/tests/lib/Files/Storage/Storage.php +++ b/tests/lib/Files/Storage/Storage.php @@ -358,6 +358,7 @@ public function testFOpen($fileName): void { $this->assertTrue($this->instance->file_exists($fileName)); $fh = $this->instance->fopen($fileName, 'r'); + $this->assertTrue(is_resource($fh)); $content = stream_get_contents($fh); $this->assertEquals(file_get_contents($textFile), $content); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php new file mode 100644 index 0000000000000..418f28faff732 --- /dev/null +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -0,0 +1,32 @@ +previewAppData = Server::get(IAppDataFactory::class)->get('preview'); + } + + #[TestDox("Test the migration from the legacy flat hierarchy to the new one")] + function testMigrationLegacyPath(): void { + $folder = $this->previewAppData->newFolder(5); + $file = $folder->newFile('64-64-crop.png', 'abcdefg'); + $job = Server::get(MovePreviewJob::class); + $this->invokePrivate($job, 'run', []); + } +} diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php new file mode 100644 index 0000000000000..68aea1bfcc167 --- /dev/null +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -0,0 +1,79 @@ +previewMapper = Server::get(PreviewMapper::class); + $this->connection = Server::get(IDBConnection::class); + } + + public function testGetAvailablePreviews() { + // Empty + $this->assertEquals([], $this->previewMapper->getAvailablePreviews([])); + + // No preview available + $this->assertEquals([42 => []], $this->previewMapper->getAvailablePreviews([42])); + + $this->createPreviewForFileId(42); + $previews = $this->previewMapper->getAvailablePreviews([42]); + $this->assertNotEmpty($previews[42]); + $this->assertNull($previews[42][0]->getLocationId()); + $this->assertNull($previews[42][0]->getBucketName()); + $this->assertNull($previews[42][0]->getObjectStoreName()); + + $this->createPreviewForFileId(43, 2); + $previews = $this->previewMapper->getAvailablePreviews([43]); + $this->assertNotEmpty($previews[43]); + $this->assertEquals('preview-2', $previews[43][0]->getBucketName()); + $this->assertEquals('default', $previews[43][0]->getObjectStoreName()); + } + + private function createPreviewForFileId(int $fileId, ?int $bucket = null) { + if ($bucket) { + $qb = $this->connection->getQueryBuilder(); + $qb->insert('preview_locations') + ->values([ + 'bucket' => $qb->createNamedParameter('preview-' . $bucket), + 'object_store' => $qb->createNamedParameter('default'), + ]); + $locationId = $qb->executeStatement(); + } + $preview = new Preview(); + $preview->setFileId($fileId); + $preview->setCrop(true); + $preview->setIsMax(true); + $preview->setWidth(100); + $preview->setHeight(100); + $preview->setSize(100); + $preview->setMtime(time()); + $preview->setMimetype(IPreview::MIMETYPE_PNG); + $preview->setEtag("abcdefg"); + + if ($locationId) { + $preview->setLocationId($locationId); + } + $this->previewMapper->insert($preview); + } +} From 6f56dcf73eb1c54920dbf81ae9bb87f1c1e8ed00 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 11 Sep 2025 14:52:34 +0200 Subject: [PATCH 07/14] fix(preview): Fix some tests Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 8 +- lib/private/Preview/BackgroundCleanupJob.php | 58 ++++---------- lib/private/Preview/Db/PreviewMapper.php | 35 +++++---- lib/private/Preview/Generator.php | 5 +- .../Preview/Storage/LocalPreviewStorage.php | 44 +++++------ .../Storage/ObjectStorePreviewStorage.php | 1 + lib/private/Preview/Storage/Root.php | 74 ------------------ lib/private/Preview/Watcher.php | 36 ++++----- lib/private/PreviewManager.php | 3 - lib/private/Server.php | 16 ++-- .../lib/Preview/BackgroundCleanupJobTest.php | 7 +- tests/lib/Preview/MovePreviewJobTest.php | 75 ++++++++++++++++++- tests/lib/Preview/PreviewMapperTest.php | 4 +- version.php | 2 +- 14 files changed, 162 insertions(+), 206 deletions(-) delete mode 100644 lib/private/Preview/Storage/Root.php diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 3541a128a46fc..67f60c900ee08 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -52,7 +52,7 @@ protected function run(mixed $argument): void { private function doRun($argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { - //return; + return; } $emptyHierarchicalPreviewFolders = false; @@ -88,7 +88,7 @@ private function doRun($argument): void { $qb = $this->connection->getQueryBuilder(); $qb->select('*') ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.jpg'))) + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -118,13 +118,13 @@ private function doRun($argument): void { } } - // Delete any left over preview directory + // Delete any leftover preview directory $this->appData->getFolder('.')->delete(); $this->appConfig->setValueBool('core', 'previewMovedDone', true); } /** - * @param array $previewFolders + * @param array $previewFolders */ private function processPreviews(array $previewFolders, bool $simplePaths): void { foreach ($previewFolders as $fileId => $previewFolder) { diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 62e3303dc4e0f..d65b740745d6c 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -35,23 +35,18 @@ public function __construct( } public function run($argument): void { - foreach ($this->getDeletedFiles() as $chunk) { - foreach ($chunk as $storage => $fileIds) { - foreach ($this->previewMapper->getByFileIds($storage, $fileIds) as $previews) { - $previewIds = []; - foreach ($previews as $preview) { - $previewIds[] = $preview->getId(); - $this->storageFactory->deletePreview($preview); - } - - $this->previewMapper->deleteByIds($storage, $previewIds); - }; + foreach ($this->getDeletedFiles() as $fileId) { + $previewIds = []; + foreach ($this->previewMapper->getByFileId($fileId) as $preview) { + $previewIds[] = $preview->getId(); + $this->storageFactory->deletePreview($preview); } + $this->previewMapper->deleteByIds($previewIds); } } /** - * @return \Iterator> + * @return \Iterator */ private function getDeletedFiles(): \Iterator { if ($this->connection->getShardDefinition('filecache')) { @@ -81,7 +76,7 @@ private function getDeletedFiles(): \Iterator { * If the related file is deleted, b.fileid will be null and the preview folder can be deleted. */ $qb = $this->connection->getQueryBuilder(); - $qb->select('p.storage_id', 'p.file_id') + $qb->select('p.file_id') ->from('previews', 'p') ->leftJoin('p', 'filecache', 'f', $qb->expr()->eq( 'p.file_id', 'f.fileid' @@ -93,30 +88,14 @@ private function getDeletedFiles(): \Iterator { } $cursor = $qb->executeQuery(); - - $lastStorageId = null; - /** @var FileId[] $tmpResult */ - $tmpResult = []; while ($row = $cursor->fetch()) { - if ($lastStorageId === null) { - $lastStorageId = $row['storage_id']; - } else if ($lastStorageId !== $row['storage_id']) { - yield [$lastStorageId => $tmpResult]; - $tmpResult = []; - $lastStorageId = $row['storage_id']; - } - $tmpResult[] = $row['file_id']; + yield $row['file_id']; } - - if (!empty($tmpResult)) { - yield [$lastStorageId => $tmpResult]; - } - $cursor->closeCursor(); } /** - * @return \Iterator> + * @return \Iterator */ private function getAllPreviewIds(int $chunkSize): \Iterator { $qb = $this->connection->getQueryBuilder(); @@ -131,20 +110,11 @@ private function getAllPreviewIds(int $chunkSize): \Iterator { $minId = 0; while (true) { $qb->setParameter('min_id', $minId); - $rows = $qb->executeQuery()->fetchAll(); - if (count($rows) > 0) { - $minId = $rows[count($rows) - 1]['id']; - $result = []; - foreach ($rows as $row) { - if (!isset($result[$row['storage_id']])) { - $result[$row['storage_id']] = []; - } - $result[$row['storage_id']][] = $row['file_id']; - } - yield $result; - } else { - break; + $cursor = $qb->executeQuery(); + while ($row = $cursor->fetch()) { + yield $row['file_id']; } + $cursor->closeCursor(); } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index e0ec05ef88f93..dba62c5a16391 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,6 +9,7 @@ namespace OC\Preview\Db; +use OC\Preview\Generator; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -28,6 +29,17 @@ public function __construct(IDBConnection $db) { parent::__construct($db, self::TABLE_NAME, Preview::class); } + /** + * @return \Generator + * @throws Exception + */ + public function getAvailablePreviewForFile(int $fileId): \Generator { + $selectQb = $this->db->getQueryBuilder(); + $this->joinLocation($selectQb) + ->where($selectQb->expr()->eq('p.file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); + } + /** * @param int[] $fileIds * @return array @@ -37,7 +49,7 @@ public function getAvailablePreviews(array $fileIds): array { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) ->where( - $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + $selectQb->expr()->in('p.file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), ); $previews = array_fill_keys($fileIds, []); foreach ($this->yieldEntities($selectQb) as $preview) { @@ -64,20 +76,13 @@ public function getPreview(int $fileId, int $width, int $height, string $mode, i } /** - * @param int[] $fileIds - * @return array + * @return \Generator */ - public function getByFileIds(array $fileIds): array { + public function getByFileId(int $fileId): \Generator { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) - ->where($selectQb->expr()->andX( - $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), - )); - $previews = array_fill_keys($fileIds, []); - foreach ($this->yieldEntities($selectQb) as $preview) { - $previews[$preview->getFileId()][] = $preview; - } - return $previews; + ->where($selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); } /** @@ -94,9 +99,9 @@ public function deleteByIds(array $previewIds): void { protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') ->from(self::TABLE_NAME, 'p') - ->join('p', 'preview_locations', 'l', $qb->expr()->eq( - 'p.location_id', 'l.id' - )); + ->leftJoin('p', 'preview_locations', 'l', $qb->expr()->eq( + 'p.location_id', 'l.id' + )); } public function getLocationId(string $bucket, string $objectStore): int { diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 44a12624c3e1a..6d3626bb2fc7f 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -349,7 +349,7 @@ private function generateProviderPreview(File $file, int $width, int $height, bo try { return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); - } catch (NotPermittedException $e) { + } catch (NotPermittedException) { throw new NotFoundException(); } } @@ -571,6 +571,9 @@ public function savePreview(File $file, int $width, int $height, bool $crop, boo } else { $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); } + if (!$size) { + throw new \RuntimeException('Unable to write preview file'); + } } catch (\Exception $e) { $this->previewMapper->delete($previewEntry); throw $e; diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index d4d7e37c0e5d8..c4577cac8bd69 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -17,7 +17,6 @@ use OCP\IConfig; class LocalPreviewStorage implements IPreviewStorage { - private const PREVIEW_DIRECTORY = '__preview'; private readonly string $rootFolder; private readonly string $instanceId; @@ -30,19 +29,18 @@ public function __construct( public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); - $this->createParentFiles($previewPath); - $file = @fopen($this->getPreviewRootFolder() . $previewPath, 'w'); - return fwrite($file, $stream); + if (!$this->createParentFiles($previewPath)) { + return false; + } + return file_put_contents($previewPath, $stream); } public function readPreview(Preview $preview) { - $previewPath = $this->constructPath($preview); - return @fopen($this->getPreviewRootFolder() . $previewPath, 'r'); + return @fopen($this->constructPath($preview), 'r'); } public function deletePreview(Preview $preview) { - $previewPath = $this->constructPath($preview); - @unlink($this->getPreviewRootFolder() . $previewPath); + @unlink($this->constructPath($preview)); } public function getPreviewRootFolder(): string { @@ -50,36 +48,28 @@ public function getPreviewRootFolder(): string { } private function constructPath(Preview $preview): string { - return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } - private function createParentFiles($path) { - ['basename' => $basename, 'dirname' => $dirname] = pathinfo($path); - $currentDir = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY; - mkdir($currentDir); - foreach (explode('/', $dirname) as $suffix) { - $currentDir .= "/$suffix"; - mkdir($currentDir); - } + private function createParentFiles(string $path): bool { + ['dirname' => $dirname] = pathinfo($path); + return mkdir($dirname, recursive: true); } public function migratePreview(Preview $preview, SimpleFile $file): void { - $instanceId = $this->config->getSystemValueString('instanceid'); - $previewPath = $this->constructPath($preview); - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; - $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; - if (file_exists($sourcePath)) { - return; // No need to migrate + // legacy flat directory + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); + if (!file_exists($sourcePath)) { + return; } - // legacy flat directory - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + $destinationPath = $this->constructPath($preview); if (file_exists($destinationPath)) { @unlink($sourcePath); // We already have a new preview, just delete the old one return; } - $this->createParentFiles($previewPath); - echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; + + $this->createParentFiles($destinationPath); $ok = rename($sourcePath, $destinationPath); if (!$ok) { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 97313fd306f1d..7963c870dd811 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -83,6 +83,7 @@ public function deletePreview(Preview $preview) { public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore $this->getObjectStoreForPreview($preview, true); + $this->previewMapper->update($preview); } /** diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php deleted file mode 100644 index 413786539629d..0000000000000 --- a/lib/private/Preview/Storage/Root.php +++ /dev/null @@ -1,74 +0,0 @@ -isMultibucketPreviewDistributionEnabled = $systemConfig->getValue('objectstore.multibucket.preview-distribution', false) === true; - } - - - public function getFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - - try { - return parent::getFolder($internalFolder); - } catch (NotFoundException $e) { - /* - * The new folder structure is not found. - * Lets try the old one - */ - } - - try { - return parent::getFolder($name); - } catch (NotFoundException $e) { - /* - * The old folder structure is not found. - * Lets try the multibucket fallback if available - */ - if ($this->isMultibucketPreviewDistributionEnabled) { - return parent::getFolder('old-multibucket/' . $internalFolder); - } - - // when there is no further fallback just throw the exception - throw $e; - } - } - - public function newFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - return parent::newFolder($internalFolder); - } - - /* - * Do not allow directory listing on this special root - * since it gets to big and time consuming - */ - public function getDirectoryListing(): array { - return []; - } - - public static function getInternalFolder(string $name): string { - return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; - } - - public function getStorageId(): int { - return $this->getAppDataRootFolder()->getStorage()->getCache()->getNumericStorageId(); - } -} diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index 21f040d8342e1..bbbbcb835faaf 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -8,11 +8,11 @@ */ namespace OC\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; use OCP\Files\FileInfo; use OCP\Files\Folder; -use OCP\Files\IAppData; use OCP\Files\Node; -use OCP\Files\NotFoundException; /** * Class Watcher @@ -22,40 +22,36 @@ * Class that will watch filesystem activity and remove previews as needed. */ class Watcher { - /** @var IAppData */ - private $appData; - /** * Watcher constructor. - * - * @param IAppData $appData */ - public function __construct(IAppData $appData) { - $this->appData = $appData; + public function __construct( + readonly private StorageFactory $storageFactory, + readonly private PreviewMapper $previewMapper, + ) { } - public function postWrite(Node $node) { + public function postWrite(Node $node): void { $this->deleteNode($node); } - protected function deleteNode(FileInfo $node) { + protected function deleteNode(FileInfo $node): void { // We only handle files if ($node instanceof Folder) { return; } - try { - if (is_null($node->getId())) { - return; - } - $folder = $this->appData->getFolder((string)$node->getId()); - $folder->delete(); - } catch (NotFoundException $e) { - //Nothing to do + if (is_null($node->getId())) { + return; + } + + [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$node->getId()]); + foreach ($previews as $preview) { + $this->storageFactory->deletePreview($preview); } } - public function versionRollback(array $data) { + public function versionRollback(array $data): void { if (isset($data['node'])) { $this->deleteNode($data['node']); } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 3bc63a55adb93..912d2b3fe5bcf 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -32,7 +32,6 @@ class PreviewManager implements IPreview { protected IConfig $config; protected IRootFolder $rootFolder; - protected IAppData $appData; protected IEventDispatcher $eventDispatcher; private ?Generator $generator = null; private GeneratorHelper $helper; @@ -59,7 +58,6 @@ class PreviewManager implements IPreview { public function __construct( IConfig $config, IRootFolder $rootFolder, - IAppData $appData, IEventDispatcher $eventDispatcher, GeneratorHelper $helper, ?string $userId, @@ -70,7 +68,6 @@ public function __construct( ) { $this->config = $config; $this->rootFolder = $rootFolder; - $this->appData = $appData; $this->eventDispatcher = $eventDispatcher; $this->helper = $helper; $this->userId = $userId; diff --git a/lib/private/Server.php b/lib/private/Server.php index fa88d8353e3b2..974838443331c 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -82,9 +82,11 @@ use OC\OCM\Model\OCMProvider; use OC\OCM\OCMDiscoveryService; use OC\OCS\DiscoveryService; +use OC\Preview\Db\PreviewMapper; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; use OC\Preview\MimeIconProvider; +use OC\Preview\Watcher; use OC\Profile\ProfileManager; use OC\Profiler\Profiler; use OC\Remote\Api\ApiFactory; @@ -291,10 +293,6 @@ public function __construct($webRoot, \OC\Config $config) { return new PreviewManager( $c->get(\OCP\IConfig::class), $c->get(IRootFolder::class), - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ), $c->get(IEventDispatcher::class), $c->get(GeneratorHelper::class), $c->get(ISession::class)->get('user_id'), @@ -306,12 +304,10 @@ public function __construct($webRoot, \OC\Config $config) { }); $this->registerAlias(IMimeIconProvider::class, MimeIconProvider::class); - $this->registerService(\OC\Preview\Watcher::class, function (ContainerInterface $c) { - return new \OC\Preview\Watcher( - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ) + $this->registerService(Watcher::class, function (ContainerInterface $c): Watcher { + return new Watcher( + $c->get(\OC\Preview\Storage\StorageFactory::class), + $c->get(PreviewMapper::class), ); }); diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index 89503085fa464..ea08b58955da0 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -82,6 +82,11 @@ protected function tearDown(): void { $this->logout(); + foreach ($this->previewMapper->getAvailablePreviews(5) as $preview) { + $this->previewStorageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + parent::tearDown(); } @@ -89,7 +94,7 @@ private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); $files = []; - for ($i = 0; $i < 11; $i++) { + foreach (range(0, 10) as $i) { $file = $userFolder->newFile($i . '.txt'); $file->putContent('hello world!'); $this->previewManager->getPreview($file); diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index 418f28faff732..a226b611c5a87 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -3,11 +3,17 @@ namespace lib\Preview; use OC\Core\BackgroundJobs\MovePreviewJob; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; +use OCP\AppFramework\Utility\ITimeFactory; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\IAppConfig; +use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\TestDox; +use PHPUnit\Framework\MockObject\MockObject; use Test\TestCase; /** @@ -16,17 +22,78 @@ #[CoversClass(MovePreviewJob::class)] class MovePreviewJobTest extends TestCase { private IAppData $previewAppData; + private PreviewMapper $previewMapper; + private IAppConfig&MockObject $appConfig; + private StorageFactory $storageFactory; public function setUp(): void { parent::setUp(); $this->previewAppData = Server::get(IAppDataFactory::class)->get('preview'); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->appConfig = $this->createMock(IAppConfig::class); + $this->appConfig->expects($this->any()) + ->method('getValueBool') + ->willReturn(false); + $this->storageFactory = Server::get(StorageFactory::class); } - #[TestDox("Test the migration from the legacy flat hierarchy to the new one")] + public function tearDown(): void { + foreach ($this->previewMapper->getAvailablePreviewForFile(5) as $preview) { + $this->storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + + foreach ($this->previewAppData->getDirectoryListing() as $folder) { + $folder->delete(); + } + } + + #[TestDox("Test the migration from the legacy flat hierarchy to the new database format")] function testMigrationLegacyPath(): void { $folder = $this->previewAppData->newFolder(5); - $file = $folder->newFile('64-64-crop.png', 'abcdefg'); - $job = Server::get(MovePreviewJob::class); - $this->invokePrivate($job, 'run', []); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + } + + private static function getInternalFolder(string $name): string { + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + function testMigrationPath(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder(5)); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder(5)); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 68aea1bfcc167..d6925641d4e1a 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -55,8 +55,8 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null) { $qb = $this->connection->getQueryBuilder(); $qb->insert('preview_locations') ->values([ - 'bucket' => $qb->createNamedParameter('preview-' . $bucket), - 'object_store' => $qb->createNamedParameter('default'), + 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), + 'object_store_name' => $qb->createNamedParameter('default'), ]); $locationId = $qb->executeStatement(); } diff --git a/version.php b/version.php index d244b03ab24ae..b4a6c28365597 100644 --- a/version.php +++ b/version.php @@ -9,7 +9,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patch level // when updating major/minor version number. -$OC_Version = [33, 0, 0, 0]; +$OC_Version = [33, 0, 0, 1]; // The human-readable string $OC_VersionString = '33.0.0 dev'; From 324b54b863ac07f93e0553462608089b7240c618 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 16 Sep 2025 11:34:41 +0200 Subject: [PATCH 08/14] refactor(preview): Cleanup the implementation of the new preview backend Signed-off-by: Carl Schwan --- build/psalm-baseline.xml | 11 - build/stubs/php-polyfill.php | 9 + core/BackgroundJobs/MovePreviewJob.php | 223 +++++++------ core/Command/Preview/Repair.php | 293 ------------------ core/Command/Preview/ResetRenderedTexts.php | 72 ++--- .../Version33000Date20250819110529.php | 3 +- core/register_command.php | 1 - lib/composer/composer/autoload_classmap.php | 3 +- lib/composer/composer/autoload_static.php | 15 +- lib/private/BackgroundJob/JobList.php | 1 - lib/private/Preview/BackgroundCleanupJob.php | 47 +-- lib/private/Preview/Db/Preview.php | 74 ++--- lib/private/Preview/Db/PreviewMapper.php | 18 +- lib/private/Preview/Generator.php | 10 +- lib/private/Preview/PreviewService.php | 101 ++++++ .../Preview/Storage/IPreviewStorage.php | 14 +- .../Preview/Storage/LocalPreviewStorage.php | 6 +- .../Storage/ObjectStorePreviewStorage.php | 11 +- .../Preview/Storage/StorageFactory.php | 12 +- lib/private/Preview/Watcher.php | 5 +- lib/private/PreviewManager.php | 1 - psalm.xml | 1 + tests/Core/Command/Preview/RepairTest.php | 153 --------- .../PrimaryObjectStoreConfigTest.php | 3 + .../lib/Preview/BackgroundCleanupJobTest.php | 41 +-- tests/lib/Preview/GeneratorTest.php | 26 +- tests/lib/Preview/MovePreviewJobTest.php | 123 +++++++- tests/lib/Preview/PreviewMapperTest.php | 17 +- tests/lib/Preview/PreviewServiceTest.php | 59 ++++ 29 files changed, 565 insertions(+), 788 deletions(-) create mode 100644 build/stubs/php-polyfill.php delete mode 100644 core/Command/Preview/Repair.php create mode 100644 lib/private/Preview/PreviewService.php delete mode 100644 tests/Core/Command/Preview/RepairTest.php create mode 100644 tests/lib/Preview/PreviewServiceTest.php diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 606cb3b5bb14b..21baeb9a33b31 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -2708,17 +2708,6 @@ timeFactory->getTime()]]> - - - - - - - - - - - diff --git a/build/stubs/php-polyfill.php b/build/stubs/php-polyfill.php new file mode 100644 index 0000000000000..606a21f8dfeb3 --- /dev/null +++ b/build/stubs/php-polyfill.php @@ -0,0 +1,9 @@ +doRun($argument); - } catch (\Throwable $exception) { - echo $exception->getMessage(); - throw $exception; - } - } - - private function doRun($argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { return; } @@ -59,14 +54,13 @@ private function doRun($argument): void { $startTime = time(); while (true) { - $previewFolders = []; - // Check new hierarchical preview folders first if (!$emptyHierarchicalPreviewFolders) { $qb = $this->connection->getQueryBuilder(); $qb->select('*') ->from('filecache') ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -74,12 +68,7 @@ private function doRun($argument): void { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; - $previewFolders[$fileId][] = $row['path']; - } - - if (!empty($previewFolders)) { - $this->processPreviews($previewFolders, false); - continue; + $this->processPreviews($fileId, false); } } @@ -89,6 +78,7 @@ private function doRun($argument): void { $qb->select('*') ->from('filecache') ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -98,18 +88,7 @@ private function doRun($argument): void { $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); $path = implode('/', $pathSplit); - if (!isset($previewFolders[$fileId])) { - $previewFolders[$fileId] = []; - } - if (!in_array($path, $previewFolders[$fileId])) { - $previewFolders[$fileId][] = $path; - } - } - - if (empty($previewFolders)) { - break; - } else { - $this->processPreviews($previewFolders, true); + $this->processPreviews($fileId, true); } // Stop if execution time is more than one hour. @@ -118,97 +97,117 @@ private function doRun($argument): void { } } - // Delete any leftover preview directory - $this->appData->getFolder('.')->delete(); + try { + // Delete any leftover preview directory + $this->appData->getFolder('.')->delete(); + } catch (NotFoundException) { + // ignore + } $this->appConfig->setValueBool('core', 'previewMovedDone', true); } /** * @param array $previewFolders */ - private function processPreviews(array $previewFolders, bool $simplePaths): void { - foreach ($previewFolders as $fileId => $previewFolder) { - $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); - $folder = $this->appData->getFolder($internalPath); - - /** - * @var list $previewFiles - */ - $previewFiles = []; - - foreach ($folder->getDirectoryListing() as $previewFile) { - /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); - - // TODO VERSION/PREFIX extraction - - $width = $nameSplit[0]; - $height = $nameSplit[1]; + private function processPreviews(int|string $fileId, bool $simplePaths): void { + $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + $folder = $this->appData->getFolder($internalPath); + + /** + * @var list $previewFiles + */ + $previewFiles = []; + + foreach ($folder->getDirectoryListing() as $previewFile) { + /** @var SimpleFile $previewFile */ + [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); + $nameSplit = explode('-', $baseName); + + $offset = 0; + $version = null; + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $version = (int)$nameSplit[0]; + } - if (isset($nameSplit[2])) { - $crop = $nameSplit[2] === 'crop'; - $max = $nameSplit[2] === 'max'; - } + $width = (int)$nameSplit[$offset + 0]; + $height = (int)$nameSplit[$offset + 1]; - $previewFiles[] = [ - 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), - ]; + $crop = false; + $max = false; + if (isset($nameSplit[$offset + 2])) { + $crop = $nameSplit[$offset + 2] === 'crop'; + $max = $nameSplit[$offset + 2] === 'max'; } - $qb = $this->connection->getQueryBuilder(); - $qb->select('*') - ->from('filecache') - ->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId))); + $previewFiles[] = [ + 'file' => $previewFile, + 'width' => $width, + 'height' => $height, + 'crop' => $crop, + 'version' => $version, + 'max' => $max, + 'extension' => $extension, + 'size' => $previewFile->getSize(), + 'mtime' => $previewFile->getMTime(), + ]; + } - $result = $qb->executeQuery(); - $result = $result->fetchAll(); - - if (count($result) > 0) { - foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); - $preview->setOldFileId($previewFile['file']->getId()); - $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCrop($previewFile['crop']); - $preview->setIsMax($previewFile['max']); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); - try { - $preview = $this->previewMapper->insert($preview); - } catch (Exception $e) { - // We already have this preview in the preview table, skip - continue; - } - - try { - $this->storageFactory->migratePreview($preview, $previewFile['file']); - $previewFile['file']->delete(); - } catch (\Exception $e) { - $this->previewMapper->delete($preview); - throw $e; - } + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->setMaxResults(1); + + $result = $qb->executeQuery(); + $result = $result->fetchAll(); + + if (count($result) > 0) { + foreach ($previewFiles as $previewFile) { + $preview = new Preview(); + $preview->setFileId((int)$fileId); + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $preview->setOldFileId($file->getId()); + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setMtime($previewFile['mtime']); + $preview->setWidth($previewFile['width']); + $preview->setHeight($previewFile['height']); + $preview->setCropped($previewFile['crop']); + $preview->setVersion($previewFile['version']); + $preview->setMax($previewFile['max']); + $preview->setEncrypted(false); + $preview->setMimetype(match ($previewFile['extension']) { + 'png' => IPreview::MIMETYPE_PNG, + 'webp' => IPreview::MIMETYPE_WEBP, + 'gif' => IPreview::MIMETYPE_GIF, + default => IPreview::MIMETYPE_JPEG, + }); + $preview->setSize($previewFile['size']); + try { + $preview = $this->previewMapper->insert($preview); + } catch (Exception $e) { + // We already have this preview in the preview table, skip + continue; + } + try { + $this->storageFactory->migratePreview($preview, $file); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->executeStatement(); + // Do not call $file->delete() as this will also delete the file from the file system + } catch (\Exception $e) { + $this->previewMapper->delete($preview); + throw $e; } } - - $this->deleteFolder($internalPath, $folder); } + + $this->deleteFolder($internalPath, $folder); } public static function getInternalFolder(string $name, bool $simplePaths): string { diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php deleted file mode 100644 index a92a4cf8ed0c2..0000000000000 --- a/core/Command/Preview/Repair.php +++ /dev/null @@ -1,293 +0,0 @@ -memoryLimit = (int)$phpIni->getBytes('memory_limit'); - $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024; - - parent::__construct(); - } - - protected function configure() { - $this - ->setName('preview:repair') - ->setDescription('distributes the existing previews into subfolders') - ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.') - ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.') - ->addOption('delete', null, InputOption::VALUE_NONE, 'Delete instead of migrating them. Usefull if too many entries to migrate.'); - } - - protected function execute(InputInterface $input, OutputInterface $output): int { - if ($this->memoryLimit !== -1) { - $limitInMiB = round($this->memoryLimit / 1024 / 1024, 1); - $thresholdInMiB = round($this->memoryTreshold / 1024 / 1024, 1); - $output->writeln("Memory limit is $limitInMiB MiB"); - $output->writeln("Memory threshold is $thresholdInMiB MiB"); - $output->writeln(''); - $memoryCheckEnabled = true; - } else { - $output->writeln('No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.'); - $output->writeln(''); - $memoryCheckEnabled = false; - } - - $dryMode = $input->getOption('dry'); - $deleteMode = $input->getOption('delete'); - - - if ($dryMode) { - $output->writeln('INFO: The migration is run in dry mode and will not modify anything.'); - $output->writeln(''); - } elseif ($deleteMode) { - $output->writeln('WARN: The migration will _DELETE_ old previews.'); - $output->writeln(''); - } - - $instanceId = $this->config->getSystemValueString('instanceid'); - - $output->writeln('This will migrate all previews from the old preview location to the new one.'); - $output->writeln(''); - - $output->writeln('Fetching previews that need to be migrated …'); - /** @var Folder $currentPreviewFolder */ - $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview"); - - $directoryListing = $currentPreviewFolder->getDirectoryListing(); - - $total = count($directoryListing); - /** - * by default there could be 0-9 a-f and the old-multibucket folder which are all fine - */ - if ($total < 18) { - $directoryListing = array_filter($directoryListing, function ($dir) { - if ($dir->getName() === 'old-multibucket') { - return false; - } - - // a-f can't be a file ID -> removing from migration - if (preg_match('!^[a-f]$!', $dir->getName())) { - return false; - } - - if (preg_match('!^[0-9]$!', $dir->getName())) { - // ignore folders that only has folders in them - if ($dir instanceof Folder) { - foreach ($dir->getDirectoryListing() as $entry) { - if (!$entry instanceof Folder) { - return true; - } - } - return false; - } - } - return true; - }); - $total = count($directoryListing); - } - - if ($total === 0) { - $output->writeln('All previews are already migrated.'); - return 0; - } - - $output->writeln("A total of $total preview files need to be migrated."); - $output->writeln(''); - $output->writeln('The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This will finish the current batch and then stop the migration. This migration can then just be started and it will continue.'); - - if ($input->getOption('batch')) { - $output->writeln('Batch mode active: migration is started right away.'); - } else { - /** @var QuestionHelper $helper */ - $helper = $this->getHelper('question'); - $question = new ConfirmationQuestion('Should the migration be started? (y/[n]) ', false); - - if (!$helper->ask($input, $output, $question)) { - return 0; - } - } - - // register the SIGINT listener late in here to be able to exit in the early process of this command - pcntl_signal(SIGINT, [$this, 'sigIntHandler']); - - $output->writeln(''); - $output->writeln(''); - $section1 = $output->section(); - $section2 = $output->section(); - $progressBar = new ProgressBar($section2, $total); - $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%'); - $time = (new \DateTime())->format('H:i:s'); - $progressBar->setMessage("$time Starting …"); - $progressBar->maxSecondsBetweenRedraws(0.2); - $progressBar->start(); - - foreach ($directoryListing as $oldPreviewFolder) { - pcntl_signal_dispatch(); - $name = $oldPreviewFolder->getName(); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Migrating previews of file with fileId $name …"); - $progressBar->display(); - - if ($this->stopSignalReceived) { - $section1->writeln("$time Stopping migration …"); - return 0; - } - if (!$oldPreviewFolder instanceof Folder) { - $section1->writeln(" Skipping non-folder $name …"); - $progressBar->advance(); - continue; - } - if ($name === 'old-multibucket') { - $section1->writeln(" Skipping fallback mount point $name …"); - $progressBar->advance(); - continue; - } - if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) { - $section1->writeln(" Skipping hex-digit folder $name …"); - $progressBar->advance(); - continue; - } - if (!preg_match('!^\d+$!', $name)) { - $section1->writeln(" Skipping non-numeric folder $name …"); - $progressBar->advance(); - continue; - } - - $newFoldername = Root::getInternalFolder($name); - - $memoryUsage = memory_get_usage(); - if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) { - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(' Stopped process 25 MB before reaching the memory limit to avoid a hard crash.'); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Reached memory limit and stopped to avoid hard crash."); - return 1; - } - - $lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId(); - try { - $section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE); - $this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - } catch (LockedException $e) { - $section1->writeln(' Skipping because it is locked - another process seems to work on this …'); - continue; - } - - $previews = $oldPreviewFolder->getDirectoryListing(); - if ($previews !== []) { - try { - $this->rootFolder->get("appdata_$instanceId/preview/$newFoldername"); - } catch (NotFoundException $e) { - $section1->writeln(" Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - $this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername"); - } - } - - foreach ($previews as $preview) { - pcntl_signal_dispatch(); - $previewName = $preview->getName(); - - if ($preview instanceof Folder) { - $section1->writeln(" Skipping folder $name/$previewName …"); - $progressBar->advance(); - continue; - } - - // Execute process - if (!$dryMode) { - // Delete preview instead of moving - if ($deleteMode) { - try { - $section1->writeln(" Delete preview/$name/$previewName", OutputInterface::VERBOSITY_VERBOSE); - $preview->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete preview at preview/$name/$previewName", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } else { - try { - $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); - } catch (\Exception $e) { - $this->logger->error("Failed to move preview from preview/$name/$previewName to preview/$newFoldername", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - } - } - - if ($oldPreviewFolder->getDirectoryListing() === []) { - $section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - try { - $oldPreviewFolder->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete empty folder preview/$name", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - - $this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - $section1->writeln(' Unlocked', OutputInterface::VERBOSITY_VERBOSE); - - $section1->writeln(" Finished migrating previews of file with fileId $name …"); - $progressBar->advance(); - } - - $progressBar->finish(); - $output->writeln(''); - return 0; - } - - protected function sigIntHandler() { - echo "\nSignal received - will finish the step and then stop the migration.\n\n\n"; - $this->stopSignalReceived = true; - } -} diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index ce874e15406f9..fb7fb9fa6200c 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -8,8 +8,8 @@ */ namespace OC\Core\Command\Preview; -use OC\Preview\Storage\Root; -use OCP\DB\QueryBuilder\IQueryBuilder; +use OC\Preview\Db\Preview; +use OC\Preview\PreviewService; use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -23,16 +23,16 @@ class ResetRenderedTexts extends Command { public function __construct( - protected IDBConnection $connection, - protected IUserManager $userManager, - protected IAvatarManager $avatarManager, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, + protected readonly IDBConnection $connection, + protected readonly IUserManager $userManager, + protected readonly IAvatarManager $avatarManager, + private readonly PreviewService $previewService, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct(); } - protected function configure() { + protected function configure(): void { $this ->setName('preview:reset-rendered-texts') ->setDescription('Deletes all generated avatars and previews of text and md files') @@ -91,7 +91,7 @@ private function getAvatarsToDelete(): \Iterator { private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; - foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) { + foreach ($this->getPreviewsToDelete() as ['path' => $filePath, 'preview' => $preview]) { $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -100,63 +100,33 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { continue; } - try { - $preview = $this->previewFolder->getFolder((string)$previewFileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue - } + $this->previewService->deletePreview($preview); } $output->writeln('Deleted ' . $previewsToDeleteCount . ' previews'); } - // Copy pasted and adjusted from - // "lib/private/Preview/BackgroundCleanupJob.php". + /** + * @return \Iterator + */ private function getPreviewsToDelete(): \Iterator { $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') + $qb->select('fileid', 'path') ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name', 'b.path') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) ->where( - $qb->expr()->andX( - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->orX( - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) - ) + $qb->expr()->orX( + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) ) ); $cursor = $qb->executeQuery(); while ($row = $cursor->fetch()) { - yield $row; + foreach ($this->previewService->getAvailablePreviewForFile($row['fileid']) as $preview) { + yield ['path' => $row['path'], 'preview' => $preview]; + } } $cursor->closeCursor(); diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 5e607024ec730..27bf9ab89b766 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -41,6 +41,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); @@ -56,7 +57,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); $table->addIndex(['file_id']); - $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'cropped', 'version'], 'previews_file_uniq_idx'); } return $schema; diff --git a/core/register_command.php b/core/register_command.php index 9fd5b9b611e0e..f6c0b9466b5cb 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -206,7 +206,6 @@ $application->add(Server::get(Command\Preview\Cleanup::class)); $application->add(Server::get(Generate::class)); - $application->add(Server::get(Command\Preview\Repair::class)); $application->add(Server::get(ResetRenderedTexts::class)); $application->add(Server::get(Add::class)); diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 5c168a25de3b6..4c2e473240bd7 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1344,7 +1344,6 @@ 'OC\\Core\\Command\\Memcache\\RedisCommand' => $baseDir . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => $baseDir . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => $baseDir . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => $baseDir . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => $baseDir . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => $baseDir . '/core/Command/Router/MatchRoute.php', @@ -1909,6 +1908,7 @@ 'OC\\Preview\\PNG' => $baseDir . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => $baseDir . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => $baseDir . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => $baseDir . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => $baseDir . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => $baseDir . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php', @@ -1919,7 +1919,6 @@ 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', - 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 423b4cc6b575f..2cb14099c01e2 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 ); public static $prefixLengthsPsr4 = array ( - 'O' => + 'O' => array ( 'OC\\Core\\' => 8, 'OC\\' => 3, 'OCP\\' => 4, ), - 'N' => + 'N' => array ( 'NCU\\' => 4, ), ); public static $prefixDirsPsr4 = array ( - 'OC\\Core\\' => + 'OC\\Core\\' => array ( 0 => __DIR__ . '/../../..' . '/core', ), - 'OC\\' => + 'OC\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/private', ), - 'OCP\\' => + 'OCP\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/public', ), - 'NCU\\' => + 'NCU\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/unstable', ), @@ -1385,7 +1385,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Memcache\\RedisCommand' => __DIR__ . '/../../..' . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => __DIR__ . '/../../..' . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => __DIR__ . '/../../..' . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => __DIR__ . '/../../..' . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => __DIR__ . '/../../..' . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => __DIR__ . '/../../..' . '/core/Command/Router/MatchRoute.php', @@ -1950,6 +1949,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\PNG' => __DIR__ . '/../../..' . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => __DIR__ . '/../../..' . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => __DIR__ . '/../../..' . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => __DIR__ . '/../../..' . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => __DIR__ . '/../../..' . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php', @@ -1960,7 +1960,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', - 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index 302bee22cd5cb..c00a51e385166 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -321,7 +321,6 @@ private function buildJob(array $row): ?IJob { /** @var IJob $job */ $job = \OCP\Server::get($row['class']); } catch (QueryException $e) { - echo $e->getMessage(); if (class_exists($row['class'])) { $class = $row['class']; $job = new $class(); diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index d65b740745d6c..257bebb1d5ded 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -8,8 +8,7 @@ */ namespace OC\Preview; -use OC\Preview\Db\PreviewMapper; -use OC\Preview\Storage\StorageFactory; +use OC\Preview\Db\Preview; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -24,8 +23,7 @@ class BackgroundCleanupJob extends TimedJob { public function __construct( ITimeFactory $timeFactory, readonly private IDBConnection $connection, - readonly private PreviewMapper $previewMapper, - readonly private StorageFactory $storageFactory, + readonly private PreviewService $previewService, readonly private bool $isCLI, ) { parent::__construct($timeFactory); @@ -37,11 +35,9 @@ public function __construct( public function run($argument): void { foreach ($this->getDeletedFiles() as $fileId) { $previewIds = []; - foreach ($this->previewMapper->getByFileId($fileId) as $preview) { - $previewIds[] = $preview->getId(); - $this->storageFactory->deletePreview($preview); + foreach ($this->previewService->getAvailablePreviewForFile($fileId) as $preview) { + $this->previewService->deletePreview($preview); } - $this->previewMapper->deleteByIds($previewIds); } } @@ -50,13 +46,12 @@ public function run($argument): void { */ private function getDeletedFiles(): \Iterator { if ($this->connection->getShardDefinition('filecache')) { - $chunks = $this->getAllPreviewIds(1000); - foreach ($chunks as $chunk) { - foreach ($chunk as $storage => $preview) { - yield [$storage => $this->findMissingSources($storage, $preview)]; + foreach ($this->previewService->getAvailableFileIds() as $availableFileIdGroup) { + $fileIds = $this->findMissingSources($availableFileIdGroup['storageId'], $availableFileIdGroup['fileIds']); + foreach ($fileIds as $fileId) { + yield $fileId; } } - return; } @@ -89,35 +84,11 @@ private function getDeletedFiles(): \Iterator { $cursor = $qb->executeQuery(); while ($row = $cursor->fetch()) { - yield $row['file_id']; + yield (int)$row['file_id']; } $cursor->closeCursor(); } - /** - * @return \Iterator - */ - private function getAllPreviewIds(int $chunkSize): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('id', 'file_id', 'storage_id') - ->from('previews') - ->where( - $qb->expr()->gt('id', $qb->createParameter('min_id')), - ) - ->orderBy('id', 'ASC') - ->setMaxResults($chunkSize); - - $minId = 0; - while (true) { - $qb->setParameter('min_id', $minId); - $cursor = $qb->executeQuery(); - while ($row = $cursor->fetch()) { - yield $row['file_id']; - } - $cursor->closeCursor(); - } - } - /** * @param FileId[] $ids * @return FileId[] diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b4ebe8b4805e5..b6f2ef2942e71 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -15,64 +15,63 @@ use OCP\IPreview; /** - * @method \int getFileId() + * Preview entity mapped to the oc_previews and oc_preview_locations table. + * + * @method int getFileId() Get the file id of the original file. * @method void setFileId(int $fileId) - * @method \int getOldFileId() // Old location in the file-cache table, for legacy compatibility - * @method void setOldFileId(int $fileId) - * @method \int getLocationId() + * @method int getStorageId() Get the storage id of the original file. + * @method void setStorageId(int $fileId) + * @method int getOldFileId() Get the old location in the file-cache table, for legacy compatibility. + * @method void setOldFileId(int $oldFileId) + * @method int getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage. * @method void setLocationId(int $locationId) - * @method \string getBucketName() - * @method \string getObjectStoreName() - * @method \int getWidth() + * @method string getBucketName() Get the bucket name where the preview is stored. This is stored in the preview_locations table. + * @method string getObjectStoreName() Get the object store name where the preview is stored. This is stored in the preview_locations table. + * @method int getWidth() Get the width of the preview. * @method void setWidth(int $width) - * @method \int getHeight() + * @method int getHeight() Get the height of the preview. * @method void setHeight(int $height) - * @method \int getMode() - * @method void setMode(int $mode) - * @method \bool getCrop() - * @method void setCrop(bool $crop) - * @method void setMimetype(int $mimetype) - * @method IPreview::MIMETYPE_* getMimetype() - * @method \int getMtime() + * @method bool isCropped() Get whether the preview is cropped or not. + * @method void setCropped(bool $cropped) + * @method void setMimetype(int $mimetype) Set the mimetype of the preview. + * @method int getMimetype() Get the mimetype of the preview. + * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) - * @method \int getSize() + * @method int getSize() Get the size of the preview. * @method void setSize(int $size) - * @method \bool getIsMax() - * @method void setIsMax(bool $max) - * @method \string getEtag() + * @method bool isMax() Get whether the preview is the biggest one which is then used to generate the smaller previews. + * @method void setMax(bool $max) + * @method string getEtag() Get the etag of the preview. * @method void setEtag(string $etag) - * @method ?\int getVersion() + * @method int|null getVersion() Get the version for files_versions_s3 * @method void setVersion(?int $version) + * @method bool|null getIs() Get the version for files_versions_s3 + * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted. + * @method void setEncrypted(bool $encrypted) + * + * @see PreviewMapper */ class Preview extends Entity { protected ?int $fileId = null; - protected ?int $oldFileId = null; - + protected ?int $storageId = null; protected ?int $locationId = null; protected ?string $bucketName = null; protected ?string $objectStoreName = null; - protected ?int $width = null; - protected ?int $height = null; - protected ?int $mimetype = null; - protected ?int $mtime = null; - protected ?int $size = null; - - protected ?bool $isMax = null; - - protected ?bool $crop = null; - + protected ?bool $max = null; + protected ?bool $cropped = null; protected ?string $etag = null; - protected ?int $version = null; + protected ?bool $encrypted = null; public function __construct() { $this->addType('fileId', Types::BIGINT); + $this->addType('storageId', Types::BIGINT); $this->addType('oldFileId', Types::BIGINT); $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); @@ -80,18 +79,19 @@ public function __construct() { $this->addType('mimetype', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); - $this->addType('isMax', Types::BOOLEAN); - $this->addType('crop', Types::BOOLEAN); + $this->addType('max', Types::BOOLEAN); + $this->addType('cropped', Types::BOOLEAN); + $this->addType('encrypted', Types::BOOLEAN); $this->addType('etag', Types::STRING); $this->addType('version', Types::BIGINT); } public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); - if ($this->getCrop()) { + if ($this->isCropped()) { $path .= '-crop'; } - if ($this->getIsMax()) { + if ($this->isMax()) { $path .= '-max'; } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index dba62c5a16391..ec3a0fa7e1fa2 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,7 +9,6 @@ namespace OC\Preview\Db; -use OC\Preview\Generator; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -123,4 +122,21 @@ public function getLocationId(string $bucket, string $objectStore): int { return $qb->getLastInsertId(); } } + + public function deleteAll(): void { + $delete = $this->db->getQueryBuilder(); + $delete->delete($this->getTableName()); + } + + /** + * @return \Generator + */ + public function getPreviews(int $lastId, int $limit = 1000): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->gt('p.id', $qb->createNamedParameter($lastId, IQueryBuilder::PARAM_INT))) + ->setMaxResults($limit); + return $this->yieldEntities($qb); + + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 6d3626bb2fc7f..c6e49a1c6ceff 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -158,7 +158,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim try { $preview = array_find($previews, fn (Preview $preview): bool => $preview->getWidth() === $width && $preview->getHeight() === $height && $preview->getMimetype() === $maxPreview->getMimetype() - && $preview->getVersion() === $previewVersion && $preview->getCrop() === $crop); + && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); @@ -300,7 +300,7 @@ private function getMaxPreview(array $previews, File $file, string $mimeType, in // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { - if ($preview->getIsMax() && ($version == $preview->getVersion())) { + if ($preview->isMax() && ($version === $preview->getVersion())) { return $preview; } } @@ -539,11 +539,13 @@ private function getExtension(string $mimeType): string { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); - $previewEntry->setIsMax($max); - $previewEntry->setCrop($crop); + $previewEntry->setMax($max); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); switch ($preview->dataMimeType()) { case 'image/jpeg': $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php new file mode 100644 index 0000000000000..67d6b011416fb --- /dev/null +++ b/lib/private/Preview/PreviewService.php @@ -0,0 +1,101 @@ +storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + + /** + * Get storageId and fileIds for which we have at least one preview. + * + * @return \Generator + */ + public function getAvailableFileIds(): \Generator { + $maxQb = $this->connection->getQueryBuilder(); + $maxQb->select($maxQb->func()->max('id')) + ->from($this->previewMapper->getTableName()) + ->groupBy('file_id'); + + $qb = $this->connection->getQueryBuilder(); + $qb->select('file_id', 'storage_id') + ->from($this->previewMapper->getTableName()) + ->where($qb->expr()->in('id', $qb->createFunction($maxQb->getSQL()))); + + $result = $qb->executeQuery(); + + $lastStorageId = -1; + /** @var int[] $fileIds */ + $fileIds = []; + + // Previews next to each others in the database are likely in the same storage, so group them + while ($row = $result->fetch()) { + if ($lastStorageId !== $row['storage_id']) { + if ($lastStorageId !== -1) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + $fileIds = []; + } + $lastStorageId = (int)$row['storage_id']; + } + $fileIds[] = (int)$row['file_id']; + } + + if (count($fileIds) > 0) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + } + } + + /** + * @return \Generator + */ + public function getAvailablePreviewForFile(int $fileId): \Generator { + yield from $this->previewMapper->getAvailablePreviewForFile($fileId); + } + + public function deleteAll(): void { + $lastId = 0; + while (true) { + $previews = $this->previewMapper->getPreviews($lastId, 1000); + $i = 0; + foreach ($previews as $preview) { + $this->deletePreview($preview); + $i++; + $lastId = $preview->getId(); + } + + if ($i !== 1000) { + break; + } + } + } + + /** + * @param int[] $fileIds + * @return array + */ + public function getAvailablePreviews(array $fileIds): array { + return $this->previewMapper->getAvailablePreviews($fileIds); + } +} diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 787518e53a40e..933fc2850b0b6 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -1,5 +1,13 @@ rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } - public function writePreview(Preview $preview, $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): false|int { $previewPath = $this->constructPath($preview); if (!$this->createParentFiles($previewPath)) { return false; @@ -35,11 +35,11 @@ public function writePreview(Preview $preview, $stream): false|int { return file_put_contents($previewPath, $stream); } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { return @fopen($this->constructPath($preview), 'r'); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { @unlink($this->constructPath($preview)); } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 7963c870dd811..0a735e5a9123d 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -15,7 +15,6 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; @@ -26,7 +25,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array> + * @var array> */ private array $objectStoreCache = []; @@ -40,7 +39,7 @@ public function __construct( $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } - public function writePreview(Preview $preview, $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): false|int { if (!is_resource($stream)) { $fh = fopen('php://temp', 'w+'); fwrite($fh, $stream); @@ -64,7 +63,7 @@ public function writePreview(Preview $preview, $stream): false|int { return $size; } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { [ 'objectPrefix' => $objectPrefix, 'store' => $store, @@ -72,12 +71,12 @@ public function readPreview(Preview $preview) { return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { [ 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - return $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } public function migratePreview(Preview $preview, SimpleFile $file): void { diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 0755bacd0b993..3051ad5869fee 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -1,5 +1,13 @@ getBackend()->writePreview($preview, $stream); } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { return $this->getBackend()->readPreview($preview); } diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index bbbbcb835faaf..ea0f72796ae5b 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -41,11 +41,12 @@ protected function deleteNode(FileInfo $node): void { return; } - if (is_null($node->getId())) { + $nodeId = $node->getId(); + if (is_null($nodeId)) { return; } - [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$node->getId()]); + [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$nodeId]); foreach ($previews as $preview) { $this->storageFactory->deletePreview($preview); } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 912d2b3fe5bcf..fb88409287260 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,7 +16,6 @@ use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; diff --git a/psalm.xml b/psalm.xml index 648b250c38b6e..abea7c5a443ea 100644 --- a/psalm.xml +++ b/psalm.xml @@ -92,6 +92,7 @@ + diff --git a/tests/Core/Command/Preview/RepairTest.php b/tests/Core/Command/Preview/RepairTest.php deleted file mode 100644 index 9b9cde6dd95af..0000000000000 --- a/tests/Core/Command/Preview/RepairTest.php +++ /dev/null @@ -1,153 +0,0 @@ -config = $this->getMockBuilder(IConfig::class) - ->getMock(); - $this->rootFolder = $this->getMockBuilder(IRootFolder::class) - ->getMock(); - $this->logger = $this->getMockBuilder(LoggerInterface::class) - ->getMock(); - $this->iniGetWrapper = $this->getMockBuilder(IniGetWrapper::class) - ->getMock(); - $this->repair = new Repair( - $this->config, - $this->rootFolder, - $this->logger, - $this->iniGetWrapper, - $this->createMock(ILockingProvider::class) - ); - $this->input = $this->createMock(InputInterface::class); - $this->input->expects($this->any()) - ->method('getOption') - ->willReturnCallback(function ($parameter) { - if ($parameter === 'batch') { - return true; - } - return null; - }); - $this->output = $this->getMockBuilder(ConsoleOutput::class) - ->onlyMethods(['section', 'writeln', 'getFormatter']) - ->getMock(); - $self = $this; - - /* We need format method to return a string */ - $outputFormatter = $this->createMock(OutputFormatterInterface::class); - $outputFormatter->method('isDecorated')->willReturn(false); - $outputFormatter->method('format')->willReturnArgument(0); - - $this->output->expects($this->any()) - ->method('getFormatter') - ->willReturn($outputFormatter); - $this->output->expects($this->any()) - ->method('writeln') - ->willReturnCallback(function ($line) use ($self): void { - $self->outputLines .= $line . "\n"; - }); - } - - public static function dataEmptyTest(): array { - /** directoryNames, expectedOutput */ - return [ - [ - [], - 'All previews are already migrated.' - ], - [ - [['name' => 'a'], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['file', 'folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - [ - [['name' => '23'], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - ]; - } - - #[\PHPUnit\Framework\Attributes\DataProvider('dataEmptyTest')] - public function testEmptyExecute($directoryNames, $expectedOutput): void { - $previewFolder = $this->getMockBuilder(Folder::class) - ->getMock(); - $directories = array_map(function ($element) { - $dir = $this->getMockBuilder(Folder::class) - ->getMock(); - $dir->expects($this->any()) - ->method('getName') - ->willReturn($element['name']); - if (isset($element['content'])) { - $list = []; - foreach ($element['content'] as $item) { - if ($item === 'file') { - $list[] = $this->getMockBuilder(Node::class) - ->getMock(); - } elseif ($item === 'folder') { - $list[] = $this->getMockBuilder(Folder::class) - ->getMock(); - } - } - $dir->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($list); - } - return $dir; - }, $directoryNames); - $previewFolder->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($directories); - $this->rootFolder->expects($this->once()) - ->method('get') - ->with('appdata_/preview') - ->willReturn($previewFolder); - - $this->repair->run($this->input, $this->output); - - $this->assertStringContainsString($expectedOutput, $this->outputLines); - } -} diff --git a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php index c13b170ca1843..07e58d56f11b9 100644 --- a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php +++ b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php @@ -212,6 +212,7 @@ public function testMultibucketOldConfig() { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -235,6 +236,7 @@ public function testSingleObjectStore() { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -270,6 +272,7 @@ public function testRoot() { $this->setConfig('objectstore', [ 'default' => 'server1', 'root' => 'server2', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index ea08b58955da0..a2c72cbad57eb 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -9,19 +9,13 @@ use OC\Files\Storage\Temporary; use OC\Preview\BackgroundCleanupJob; -use OC\Preview\Db\Preview; -use OC\Preview\Db\PreviewMapper; -use OC\Preview\Storage\Root; -use OC\Preview\Storage\StorageFactory; +use OC\Preview\PreviewService; use OC\PreviewManager; -use OC\SystemConfig; use OCP\App\IAppManager; use OCP\AppFramework\Utility\ITimeFactory; -use OCP\Files\AppData\IAppDataFactory; use OCP\Files\File; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; -use OCP\Files\NotFoundException; use OCP\IDBConnection; use OCP\IPreview; use OCP\Server; @@ -45,8 +39,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { private IRootFolder $rootFolder; private IMimeTypeLoader $mimeTypeLoader; private ITimeFactory $timeFactory; - private PreviewMapper $previewMapper; - private StorageFactory $previewStorageFactory; + private PreviewService $previewService; protected function setUp(): void { parent::setUp(); @@ -70,8 +63,7 @@ protected function setUp(): void { $this->rootFolder = Server::get(IRootFolder::class); $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); $this->timeFactory = Server::get(ITimeFactory::class); - $this->previewMapper = Server::get(PreviewMapper::class); - $this->previewStorageFactory = Server::get(StorageFactory::class); + $this->previewService = Server::get(PreviewService::class); } protected function tearDown(): void { @@ -82,9 +74,8 @@ protected function tearDown(): void { $this->logout(); - foreach ($this->previewMapper->getAvailablePreviews(5) as $preview) { - $this->previewStorageFactory->deletePreview($preview); - $this->previewMapper->delete($preview); + foreach ($this->previewService->getAvailablePreviewForFile(5) as $preview) { + $this->previewService->deletePreview($preview); } parent::tearDown(); @@ -104,8 +95,8 @@ private function setup11Previews(): array { return $files; } - private function countPreviews(PreviewMapper $previewMapper, array $fileIds): int { - $previews = $previewMapper->getAvailablePreviews($fileIds); + private function countPreviews(PreviewService $previewService, array $fileIds): int { + $previews = $previewService->getAvailablePreviews($fileIds); return array_reduce($previews, fn (int $result, array $previews) => $result + count($previews), 0); } @@ -113,18 +104,18 @@ public function testCleanupSystemCron(): void { $files = $this->setup11Previews(); $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, true); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } public function testCleanupAjax(): void { @@ -134,20 +125,20 @@ public function testCleanupAjax(): void { $files = $this->setup11Previews(); $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, false); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(1, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(1, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 91fa353746814..54d28747cf3d5 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -16,8 +16,6 @@ use OCP\Files\File; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -84,19 +82,21 @@ public function testGetCachedPreview(): void { $maxPreview = new Preview(); $maxPreview->setWidth(1000); $maxPreview->setHeight(1000); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setCrop(false); + $maxPreview->setCropped(false); + $maxPreview->setStorageId(1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); $previewFile = new Preview(); $previewFile->setWidth(256); $previewFile->setHeight(256); - $previewFile->setIsMax(false); + $previewFile->setMax(false); $previewFile->setSize(1000); $previewFile->setVersion(-1); - $previewFile->setCrop(false); + $previewFile->setCropped(false); + $previewFile->setStorageId(1); $previewFile->setMimetype(IPreview::MIMETYPE_PNG); $this->previewMapper->method('getAvailablePreviews') @@ -190,7 +190,7 @@ public function testGetNewPreview(): void { $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -210,7 +210,7 @@ public function testGetNewPreview(): void { $this->assertSame('my resized data', $data); return 1000; } - $this->fail("file name is wrong:". $preview->getName()); + $this->fail('file name is wrong:' . $preview->getName()); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -238,7 +238,7 @@ public function testInvalidMimeType(): void { $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -262,7 +262,7 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -270,9 +270,9 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $previewFile = new Preview(); $previewFile->setWidth(1024); $previewFile->setHeight(512); - $previewFile->setIsMax(false); + $previewFile->setMax(false); $previewFile->setSize(1000); - $previewFile->setCrop(true); + $previewFile->setCropped(true); $previewFile->setVersion(-1); $previewFile->setMimetype(IPreview::MIMETYPE_PNG); @@ -380,7 +380,7 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $maxPreview = new Preview(); $maxPreview->setWidth($maxX); $maxPreview->setHeight($maxY); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index a226b611c5a87..cd9f325365463 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -1,17 +1,26 @@ appConfig->expects($this->any()) ->method('getValueBool') ->willReturn(false); + $this->appConfig->expects($this->any()) + ->method('setValueBool') + ->willReturn(true); $this->storageFactory = Server::get(StorageFactory::class); + $this->previewService = Server::get(PreviewService::class); + $this->db = Server::get(IDBConnection::class); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); + + $qb = $this->db->getQueryBuilder(); + $qb->insert('filecache') + ->values([ + 'fileid' => $qb->createNamedParameter(5), + 'storage' => $qb->createNamedParameter(1), + 'path' => $qb->createNamedParameter('test/abc'), + 'path_hash' => $qb->createNamedParameter(md5('test')), + 'parent' => $qb->createNamedParameter(0), + 'name' => $qb->createNamedParameter('abc'), + 'mimetype' => $qb->createNamedParameter(0), + 'size' => $qb->createNamedParameter(1000), + 'mtime' => $qb->createNamedParameter(1000), + 'storage_mtime' => $qb->createNamedParameter(1000), + 'encrypted' => $qb->createNamedParameter(0), + 'unencrypted_size' => $qb->createNamedParameter(0), + 'etag' => $qb->createNamedParameter('abcdefg'), + 'permissions' => $qb->createNamedParameter(0), + 'checksum' => $qb->createNamedParameter('abcdefg'), + ])->executeStatement(); } public function tearDown(): void { - foreach ($this->previewMapper->getAvailablePreviewForFile(5) as $preview) { - $this->storageFactory->deletePreview($preview); - $this->previewMapper->delete($preview); - } - foreach ($this->previewAppData->getDirectoryListing() as $folder) { $folder->delete(); } + $this->previewService->deleteAll(); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); } - #[TestDox("Test the migration from the legacy flat hierarchy to the new database format")] - function testMigrationLegacyPath(): void { - $folder = $this->previewAppData->newFolder(5); + #[TestDox('Test the migration from the legacy flat hierarchy to the new database format')] + public function testMigrationLegacyPath(): void { + $folder = $this->previewAppData->newFolder('5'); $folder->newFile('64-64-crop.jpg', 'abcdefg'); $folder->newFile('128-128-crop.png', 'abcdefg'); $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); @@ -63,6 +104,7 @@ function testMigrationLegacyPath(): void { $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), + Server::get(IRootFolder::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); @@ -75,12 +117,12 @@ private static function getInternalFolder(string $name): string { } #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] - function testMigrationPath(): void { - $folder = $this->previewAppData->newFolder(self::getInternalFolder(5)); + public function testMigrationPath(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); $folder->newFile('64-64-crop.jpg', 'abcdefg'); $folder->newFile('128-128-crop.png', 'abcdefg'); - $folder = $this->previewAppData->getFolder(self::getInternalFolder(5)); + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(2, count($folder->getDirectoryListing())); $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); @@ -90,10 +132,65 @@ function testMigrationPath(): void { $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), + Server::get(IRootFolder::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + public function testMigrationPathWithVersion(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); + // No version + $folder->newFile('128-128-crop.png', 'abcdefg'); + $folder->newFile('256-256-max.png', 'abcdefg'); + $folder->newFile('128-128.png', 'abcdefg'); + + // Version 1000 + $folder->newFile('1000-128-128-crop.png', 'abcdefg'); + $folder->newFile('1000-256-256-max.png', 'abcdefg'); + $folder->newFile('1000-128-128.png', 'abcdefg'); + + // Version 1001 + $folder->newFile('1001-128-128-crop.png', 'abcdefg'); + $folder->newFile('1001-256-256-max.png', 'abcdefg'); + $folder->newFile('1001-128-128.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); + $this->assertEquals(9, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IRootFolder::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $previews = iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)); + $this->assertEquals(9, count($previews)); + + $nameVersionMapping = []; + foreach ($previews as $preview) { + $nameVersionMapping[$preview->getName()] = $preview->getVersion(); + } + + $this->assertEquals([ + '1000-128-128.png' => 1000, + '1000-128-128-crop.png' => 1000, + '1000-256-256-max.png' => 1000, + '1001-128-128.png' => 1001, + '1001-128-128-crop.png' => 1001, + '1001-256-256-max.png' => 1001, + '128-128.png' => -1, + '128-128-crop.png' => -1, + '256-256-max.png' => -1, + ], $nameVersionMapping); + } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index d6925641d4e1a..6f59c6f880297 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -29,7 +29,7 @@ public function setUp(): void { $this->connection = Server::get(IDBConnection::class); } - public function testGetAvailablePreviews() { + public function testGetAvailablePreviews(): void { // Empty $this->assertEquals([], $this->previewMapper->getAvailablePreviews([])); @@ -50,7 +50,8 @@ public function testGetAvailablePreviews() { $this->assertEquals('default', $previews[43][0]->getObjectStoreName()); } - private function createPreviewForFileId(int $fileId, ?int $bucket = null) { + private function createPreviewForFileId(int $fileId, ?int $bucket = null): void { + $locationId = null; if ($bucket) { $qb = $this->connection->getQueryBuilder(); $qb->insert('preview_locations') @@ -58,20 +59,22 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null) { 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), 'object_store_name' => $qb->createNamedParameter('default'), ]); - $locationId = $qb->executeStatement(); + $qb->executeStatement(); + $locationId = $qb->getLastInsertId(); } $preview = new Preview(); $preview->setFileId($fileId); - $preview->setCrop(true); - $preview->setIsMax(true); + $preview->setStorageId(1); + $preview->setCropped(true); + $preview->setMax(true); $preview->setWidth(100); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); $preview->setMimetype(IPreview::MIMETYPE_PNG); - $preview->setEtag("abcdefg"); + $preview->setEtag('abcdefg'); - if ($locationId) { + if ($locationId !== null) { $preview->setLocationId($locationId); } $this->previewMapper->insert($preview); diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php new file mode 100644 index 0000000000000..01bcff29a5ad2 --- /dev/null +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -0,0 +1,59 @@ +previewService = Server::get(PreviewService::class); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->previewService->deleteAll(); + } + + public function tearDown(): void { + $this->previewService->deleteAll(); + } + + public function testGetAvailableFileIds(): void { + foreach (range(1, 20) as $i) { + $preview = new Preview(); + $preview->setFileId($i % 10); + $preview->setStorageId(1); + $preview->setWidth($i); + $preview->setHeight($i); + $preview->setMax(true); + $preview->setCropped(true); + $preview->setEncrypted(false); + $preview->setMimetype(IPreview::MIMETYPE_JPEG); + $preview->setEtag('abc'); + $preview->setMtime((new \DateTime())->getTimestamp()); + $preview->setSize(0); + $this->previewMapper->insert($preview); + } + + $files = iterator_to_array($this->previewService->getAvailableFileIds()); + $this->assertCount(1, $files); + $this->assertCount(10, $files[0]['fileIds']); + } +} From bfc7d5dd9fac04124db1b65e9f22d9f33a5e5d12 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 25 Sep 2025 14:25:47 +0200 Subject: [PATCH 09/14] feat(preview): Implement scanning for previews This work similarly to the move preview job to migrate the previews to the new DB table and also reuse some code. So when we are finding files in appdata/preview, try adding them to the oc_previews table and delete them from the oc_filecache table. Signed-off-by: Carl Schwan --- apps/files/lib/Command/ScanAppData.php | 39 +++-- build/psalm-baseline.xml | 4 +- core/BackgroundJobs/MovePreviewJob.php | 62 ++------ lib/private/Files/Cache/Scanner.php | 9 +- lib/private/Preview/Db/Preview.php | 35 +++++ .../Preview/Storage/IPreviewStorage.php | 2 + .../Preview/Storage/LocalPreviewStorage.php | 135 +++++++++++++++++- .../Storage/ObjectStorePreviewStorage.php | 4 + .../Preview/Storage/StorageFactory.php | 7 +- lib/public/Preview/IVersionedPreviewFile.php | 6 +- 10 files changed, 232 insertions(+), 71 deletions(-) diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 385e0624b3a92..23604a82df9b5 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -12,6 +12,7 @@ use OC\DB\ConnectionAdapter; use OC\Files\Utils\Scanner; use OC\ForbiddenException; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -32,10 +33,12 @@ class ScanAppData extends Base { protected int $foldersCounter = 0; protected int $filesCounter = 0; + protected int $previewsCounter = -1; public function __construct( protected IRootFolder $rootFolder, protected IConfig $config, + private StorageFactory $previewStorage, ) { parent::__construct(); } @@ -51,9 +54,12 @@ protected function configure(): void { } protected function scanFiles(OutputInterface $output, string $folder): int { - if ($folder === 'preview') { - $output->writeln('Scanning the preview folder is not supported.'); - return self::FAILURE; + if ($folder === 'preview' || $folder === '') { + $this->previewsCounter = $this->previewStorage->scan(); + + if ($folder === 'preview') { + return self::SUCCESS; + } } try { @@ -139,7 +145,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->initTools(); $exitCode = $this->scanFiles($output, $folder); - if ($exitCode === 0) { + if ($exitCode === self::SUCCESS) { $this->presentStats($output); } return $exitCode; @@ -167,7 +173,7 @@ protected function initTools(): void { * * @throws \ErrorException */ - public function exceptionErrorHandler($severity, $message, $file, $line) { + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; @@ -178,10 +184,12 @@ public function exceptionErrorHandler($severity, $message, $file, $line) { protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); - - $headers = [ - 'Folders', 'Files', 'Elapsed time' - ]; + if ($this->previewsCounter !== -1) { + $headers[] = 'Previews'; + } + $headers[] = 'Folders'; + $headers[] = 'Files'; + $headers[] = 'Elapsed time'; $this->showSummary($headers, null, $output); } @@ -192,14 +200,15 @@ protected function presentStats(OutputInterface $output): void { * @param string[] $headers * @param string[] $rows */ - protected function showSummary($headers, $rows, OutputInterface $output): void { + protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void { $niceDate = $this->formatExecTime(); if (!$rows) { - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $niceDate, - ]; + if ($this->previewsCounter !== -1) { + $rows[] = $this->previewsCounter; + } + $rows[] = $this->foldersCounter; + $rows[] = $this->filesCounter; + $rows[] = $niceDate; } $table = new Table($output); $table diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 21baeb9a33b31..8be3f1e9c1d13 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1240,9 +1240,11 @@ + + + - diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 22db7aea90dbd..764561a1ba828 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -82,13 +82,18 @@ protected function run(mixed $argument): void { ->setMaxResults(100); $result = $qb->executeQuery(); + $foundOldPreview = false; while ($row = $result->fetch()) { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); - $path = implode('/', $pathSplit); $this->processPreviews($fileId, true); + $foundOldPreview = true; + } + + if (!$foundOldPreview) { + break; } // Stop if execution time is more than one hour. @@ -114,44 +119,21 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $version = null; - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $version = (int)$nameSplit[0]; - } - - $width = (int)$nameSplit[$offset + 0]; - $height = (int)$nameSplit[$offset + 1]; - - $crop = false; - $max = false; - if (isset($nameSplit[$offset + 2])) { - $crop = $nameSplit[$offset + 2] === 'crop'; - $max = $nameSplit[$offset + 2] === 'max'; - } + $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview->setSize($previewFile->getSize()); + $preview->setMtime($previewFile->getMtime()); + $preview->setOldFileId($previewFile->getId()); + $preview->setEncrypted(false); $previewFiles[] = [ 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'version' => $version, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), + 'preview' => $preview, ]; } @@ -166,27 +148,11 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); + $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; - $preview->setOldFileId($file->getId()); $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCropped($previewFile['crop']); - $preview->setVersion($previewFile['version']); - $preview->setMax($previewFile['max']); - $preview->setEncrypted(false); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index b067f70b8cb51..75ca9da0abe15 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; + private string $previewFolder; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); @@ -75,6 +77,7 @@ public function __construct(\OC\Files\Storage\Storage $storage) { $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); $this->lockingProvider = \OC::$server->get(ILockingProvider::class); $this->connection = \OC::$server->get(IDBConnection::class); + $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; } /** @@ -318,7 +321,6 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc try { $data = $this->scanFile($path, $reuse, -1, lock: $lock); - if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; @@ -413,6 +415,11 @@ protected function scanChildren(string $path, $recursive, int $reuse, int $folde $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + if (str_starts_with($path, $this->previewFolder)) { + // Preview scanning is handled in LocalPreviewStorage + return 0; + } + foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b6f2ef2942e71..3f10328fd8ee2 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -86,6 +86,41 @@ public function __construct() { $this->addType('version', Types::BIGINT); } + public static function fromPath(string $path): Preview { + $preview = new self(); + $preview->setFileId((int)basename(dirname($path))); + + $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + + [0 => $baseName, 1 => $extension] = explode('.', $fileName); + $preview->setMimetype(match ($extension) { + 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, + 'png' => IPreview::MIMETYPE_PNG, + 'gif' => IPreview::MIMETYPE_GIF, + 'webp' => IPreview::MIMETYPE_WEBP, + default => IPreview::MIMETYPE_JPEG, + }); + $nameSplit = explode('-', $baseName); + + $offset = 0; + $preview->setVersion(null); + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $preview->setVersion((int)$nameSplit[0]); + } + + $preview->setWidth((int)$nameSplit[$offset + 0]); + $preview->setHeight((int)$nameSplit[$offset + 1]); + + $preview->setCropped(false); + $preview->setMax(false); + if (isset($nameSplit[$offset + 2])) { + $preview->setCropped($nameSplit[$offset + 2] === 'crop'); + $preview->setMax($nameSplit[$offset + 2] === 'max'); + } + return $preview; + } + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 933fc2850b0b6..5646432683848 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -36,4 +36,6 @@ public function deletePreview(Preview $preview): void; * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + + public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 59a4248e92be5..3bd1fe2ccb98d 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -14,7 +14,13 @@ use OC; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OCP\DB\Exception; +use OCP\IAppConfig; use OCP\IConfig; +use OCP\IDBConnection; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; class LocalPreviewStorage implements IPreviewStorage { private readonly string $rootFolder; @@ -22,6 +28,10 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, + private readonly StorageFactory $previewStorage, + private readonly IAppConfig $appConfig, + private readonly IDBConnection $connection, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -52,8 +62,9 @@ private function constructPath(Preview $preview): string { } private function createParentFiles(string $path): bool { - ['dirname' => $dirname] = pathinfo($path); - return mkdir($dirname, recursive: true); + $dirname = dirname($path); + @mkdir($dirname, recursive: true); + return is_dir($dirname); } public function migratePreview(Preview $preview, SimpleFile $file): void { @@ -75,4 +86,124 @@ public function migratePreview(Preview $preview, SimpleFile $file): void { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } } + + public function scan(): int { + $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); + + $scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder()); + $previewsFound = 0; + foreach (new RecursiveIteratorIterator($scanner) as $file) { + if ($file->isFile()) { + $preview = Preview::fromPath((string)$file); + try { + $preview->setSize($file->getSize()); + $preview->setMtime($file->getMtime()); + $preview->setEncrypted(false); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) + ->setMaxResults(1) + ->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + // original file is deleted + @unlink($file->getRealPath()); + continue; + } + + if ($checkForFileCache) { + $relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath()); + $rowAffected = $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath)))) + ->executeStatement(); + if ($rowAffected > 0) { + $this->deleteParentsFromFileCache(dirname($relativePath)); + } + } + + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + + // try to insert, if that fails the preview is already in the DB + $this->previewMapper->insert($preview); + + // Move old flat preview to new format + $this->previewStorage->migratePreview($preview, $file); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + $previewsFound++; + } + } + + return $previewsFound; + } + + private function deleteParentsFromFileCache(string $dirname): void { + $qb = $this->connection->getQueryBuilder(); + + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname)))) + ->setMaxResults(1) + ->runAcrossAllShards() + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + return; + } + + $this->connection->beginTransaction(); + + $parentId = $result[0]['parent']; + $fileId = $result[0]['fileid']; + $storage = $result[0]['storage']; + + try { + while (true) { + $qb = $this->connection->getQueryBuilder(); + $childs = $qb->select('fileid', 'path', 'storage') + ->from('filecache') + ->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (!empty($childs)) { + break; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $result[0]['storage']) + ->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId))) + ->setMaxResults(1) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + break; + } + + $fileId = $parentId; + $parentId = $result[0]['parent']; + } + } finally { + $this->connection->commit(); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 0a735e5a9123d..faa637637791f 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -149,4 +149,8 @@ public function getObjectPrefix(Preview $preview, array $config): string { return 'uri:oid:preview:'; } } + + public function scan(): int { + return 0; + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3051ad5869fee..3438534d07ed2 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -15,6 +15,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\IConfig; +use OCP\Server; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -46,7 +47,7 @@ private function getBackend(): IPreviewStorage { if ($this->objectStoreConfig->hasObjectStore()) { $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { - $this->backend = new LocalPreviewStorage($this->config); + $this->backend = Server::get(LocalPreviewStorage::class); } return $this->backend; @@ -55,4 +56,8 @@ private function getBackend(): IPreviewStorage { public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + + public function scan(): int { + return $this->getBackend()->scan(); + } } diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 9266b1ac06700..7d68fe8d15e87 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -11,15 +11,15 @@ * Marks files that should keep multiple preview "versions" for the same file id * * Examples of this are files where the storage backend provides versioning, for those - * files, we dont have fileids for the different versions but still need to be able to generate + * files, we don't have fileIds for the different versions but still need to be able to generate * previews for all versions * * @since 17.0.0 */ interface IVersionedPreviewFile { /** - * @return string + * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(): string; + public function getPreviewVersion(); } From 58023782b69637221272718699eb2519c530279b Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 26 Sep 2025 12:26:43 +0200 Subject: [PATCH 10/14] feat(preview): Store original file mimetype in preview table Allow to quickly query all the files from a specific mimetype like in the ResetRenderedTexts command. Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 4 +- core/Command/Preview/ResetRenderedTexts.php | 37 +++++-------------- lib/private/Preview/Db/Preview.php | 5 +++ lib/private/Preview/Db/PreviewMapper.php | 15 ++++++++ lib/private/Preview/Generator.php | 3 ++ lib/private/Preview/PreviewService.php | 10 ++++- .../Preview/Storage/LocalPreviewStorage.php | 3 +- lib/private/PreviewManager.php | 2 + tests/lib/Preview/GeneratorTest.php | 6 ++- tests/lib/Preview/PreviewMapperTest.php | 1 + tests/lib/Preview/PreviewServiceTest.php | 1 + 11 files changed, 55 insertions(+), 32 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 764561a1ba828..82acea82c7a9b 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -24,7 +24,6 @@ use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; -use OCP\IPreview; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -138,7 +137,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { } $qb = $this->connection->getQueryBuilder(); - $qb->select('*') + $qb->select('storage', 'etag', 'mimetype') ->from('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) ->setMaxResults(1); @@ -153,6 +152,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { $file = $previewFile['file']; $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); + $preview->setSourceMimetype($result[0]['mimetype']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index fb7fb9fa6200c..b80f0f6ffceab 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -67,9 +67,7 @@ private function deleteAvatars(OutputInterface $output, bool $dryMode): void { try { $avatar->remove(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { + } catch (NotFoundException|NotPermittedException) { // continue } } @@ -91,8 +89,8 @@ private function getAvatarsToDelete(): \Iterator { private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; - foreach ($this->getPreviewsToDelete() as ['path' => $filePath, 'preview' => $preview]) { - $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); + foreach ($this->getPreviewsToDelete() as $preview) { + $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -107,28 +105,13 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { } /** - * @return \Iterator + * @return \Generator */ - private function getPreviewsToDelete(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('fileid', 'path') - ->from('filecache') - ->where( - $qb->expr()->orX( - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) - ) - ); - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - foreach ($this->previewService->getAvailablePreviewForFile($row['fileid']) as $preview) { - yield ['path' => $row['path'], 'preview' => $preview]; - } - } - - $cursor->closeCursor(); + private function getPreviewsToDelete(): \Generator { + return $this->previewService->getPreviewsForMimeTypes([ + $this->mimeTypeLoader->getId('text/plain'), + $this->mimeTypeLoader->getId('text/markdown'), + $this->mimeTypeLoader->getId('text/x-markdown'), + ]); } } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index 3f10328fd8ee2..fe30da6589c9b 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -35,6 +35,8 @@ * @method void setCropped(bool $cropped) * @method void setMimetype(int $mimetype) Set the mimetype of the preview. * @method int getMimetype() Get the mimetype of the preview. + * @method void setSourceMimetype(int $sourceMimetype) Set the mimetype of the source file. + * @method int getSourceMimetype() Get the mimetype of the source file. * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) * @method int getSize() Get the size of the preview. @@ -61,6 +63,8 @@ class Preview extends Entity { protected ?int $width = null; protected ?int $height = null; protected ?int $mimetype = null; + + protected ?int $sourceMimetype = null; protected ?int $mtime = null; protected ?int $size = null; protected ?bool $max = null; @@ -77,6 +81,7 @@ public function __construct() { $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); + $this->addType('sourceMimetype', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); $this->addType('max', Types::BOOLEAN); diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index ec3a0fa7e1fa2..d4ef242eb21ff 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -139,4 +139,19 @@ public function getPreviews(int $lastId, int $limit = 1000): \Generator { return $this->yieldEntities($qb); } + + /** + * @param int[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->orX( + ...array_map(function (int $mimeType) use ($qb) { + return $qb->expr()->eq('source_mimetype', $qb->createNamedParameter($mimeType, IQueryBuilder::PARAM_INT)); + }, $mimeTypes) + )); + return $this->yieldEntities($qb); + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index c6e49a1c6ceff..1743ca97e0d4f 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,6 +12,7 @@ use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -38,6 +39,7 @@ public function __construct( private LoggerInterface $logger, private PreviewMapper $previewMapper, private StorageFactory $storageFactory, + private IMimeTypeLoader $mimeTypeLoader, ) { } @@ -541,6 +543,7 @@ public function savePreview(File $file, int $width, int $height, bool $crop, boo $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); + $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); $previewEntry->setHeight($height); $previewEntry->setVersion($version); $previewEntry->setMax($max); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php index 67d6b011416fb..0e7c66dc12dd5 100644 --- a/lib/private/Preview/PreviewService.php +++ b/lib/private/Preview/PreviewService.php @@ -71,7 +71,15 @@ public function getAvailableFileIds(): \Generator { * @return \Generator */ public function getAvailablePreviewForFile(int $fileId): \Generator { - yield from $this->previewMapper->getAvailablePreviewForFile($fileId); + return $this->previewMapper->getAvailablePreviewForFile($fileId); + } + + /** + * @param int[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + return $this->previewMapper->getPreviewsForMimeTypes($mimeTypes); } public function deleteAll(): void { diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 3bd1fe2ccb98d..9af1b59417532 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -101,7 +101,7 @@ public function scan(): int { $preview->setEncrypted(false); $qb = $this->connection->getQueryBuilder(); - $result = $qb->select('*') + $result = $qb->select('storage', 'etag', 'mimetype') ->from('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) ->setMaxResults(1) @@ -127,6 +127,7 @@ public function scan(): int { $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); + $preview->setSourceMimetype($result[0]['mimetype']); // try to insert, if that fails the preview is already in the DB $this->previewMapper->insert($preview); diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index fb88409287260..480408d772436 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,6 +16,7 @@ use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; @@ -139,6 +140,7 @@ private function getGenerator(): Generator { $this->container->get(LoggerInterface::class), $this->container->get(PreviewMapper::class), $this->container->get(StorageFactory::class), + $this->container->get(IMimeTypeLoader::class), ); } return $this->generator; diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 54d28747cf3d5..640c2ce8b36bf 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -14,6 +14,7 @@ use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\IConfig; @@ -34,6 +35,7 @@ class GeneratorTest extends TestCase { private LoggerInterface&MockObject $logger; private StorageFactory&MockObject $storageFactory; private PreviewMapper&MockObject $previewMapper; + private IMimeTypeLoader&MockObject $mimeTypeLoader; protected function setUp(): void { parent::setUp(); @@ -45,6 +47,7 @@ protected function setUp(): void { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); + $this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class); $this->generator = new Generator( $this->config, @@ -53,7 +56,8 @@ protected function setUp(): void { $this->eventDispatcher, $this->logger, $this->previewMapper, - $this->storageFactory + $this->storageFactory, + $this->mimetypeLoader, ); } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 6f59c6f880297..e4b2b1ab21ad4 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -68,6 +68,7 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null): void $preview->setCropped(true); $preview->setMax(true); $preview->setWidth(100); + $preview->setSourceMimetype(1); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index 01bcff29a5ad2..f9a8d88f664b2 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -43,6 +43,7 @@ public function testGetAvailableFileIds(): void { $preview->setWidth($i); $preview->setHeight($i); $preview->setMax(true); + $preview->setSourceMimetype(1); $preview->setCropped(true); $preview->setEncrypted(false); $preview->setMimetype(IPreview::MIMETYPE_JPEG); From bd001c9524ffc0ffd81a0b62571678784af00148 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 26 Sep 2025 14:22:38 +0200 Subject: [PATCH 11/14] refactor: Use Override annotation in new preview code Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 2 + core/Command/Preview/ResetRenderedTexts.php | 3 ++ .../Preview/Storage/LocalPreviewStorage.php | 6 +++ .../Storage/ObjectStorePreviewStorage.php | 6 +++ lib/private/Preview/Storage/PreviewFile.php | 45 +++++-------------- .../Preview/Storage/StorageFactory.php | 6 +++ 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 82acea82c7a9b..0eaf409e433f4 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -24,6 +24,7 @@ use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; +use Override; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -44,6 +45,7 @@ public function __construct( $this->setInterval(24 * 60 * 60); } + #[Override] protected function run(mixed $argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { return; diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index b80f0f6ffceab..6ec26ad6ac340 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -16,6 +16,7 @@ use OCP\IAvatarManager; use OCP\IDBConnection; use OCP\IUserManager; +use Override; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -32,6 +33,7 @@ public function __construct( parent::__construct(); } + #[Override] protected function configure(): void { $this ->setName('preview:reset-rendered-texts') @@ -39,6 +41,7 @@ protected function configure(): void { ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.'); } + #[Override] protected function execute(InputInterface $input, OutputInterface $output): int { $dryMode = $input->getOption('dry'); diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 9af1b59417532..8d85897115b3f 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -19,6 +19,7 @@ use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; +use Override; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; @@ -37,6 +38,7 @@ public function __construct( $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { $previewPath = $this->constructPath($preview); if (!$this->createParentFiles($previewPath)) { @@ -45,10 +47,12 @@ public function writePreview(Preview $preview, mixed $stream): false|int { return file_put_contents($previewPath, $stream); } + #[Override] public function readPreview(Preview $preview): mixed { return @fopen($this->constructPath($preview), 'r'); } + #[Override] public function deletePreview(Preview $preview): void { @unlink($this->constructPath($preview)); } @@ -67,6 +71,7 @@ private function createParentFiles(string $path): bool { return is_dir($dirname); } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); @@ -87,6 +92,7 @@ public function migratePreview(Preview $preview, SimpleFile $file): void { } } + #[Override] public function scan(): int { $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index faa637637791f..88470802667f7 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -17,6 +17,7 @@ use OC\Preview\Db\PreviewMapper; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; +use Override; /** * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig @@ -39,6 +40,7 @@ public function __construct( $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { if (!is_resource($stream)) { $fh = fopen('php://temp', 'w+'); @@ -63,6 +65,7 @@ public function writePreview(Preview $preview, mixed $stream): false|int { return $size; } + #[Override] public function readPreview(Preview $preview): mixed { [ 'objectPrefix' => $objectPrefix, @@ -71,6 +74,7 @@ public function readPreview(Preview $preview): mixed { return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } + #[Override] public function deletePreview(Preview $preview): void { [ 'objectPrefix' => $objectPrefix, @@ -79,6 +83,7 @@ public function deletePreview(Preview $preview): void { $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore $this->getObjectStoreForPreview($preview, true); @@ -150,6 +155,7 @@ public function getObjectPrefix(Preview $preview, array $config): string { } } + #[Override] public function scan(): int { return 0; } diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index c9381ce933cda..8c9852673cce2 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -13,6 +13,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\Files\SimpleFS\ISimpleFile; +use Override; class PreviewFile implements ISimpleFile { public function __construct( @@ -22,80 +23,58 @@ public function __construct( ) { } - /** - * @inheritDoc - */ + #[Override] public function getName(): string { return $this->preview->getName(); } - /** - * @inheritDoc - */ + #[Override] public function getSize(): int|float { return $this->preview->getSize(); } - /** - * @inheritDoc - */ + #[Override] public function getETag(): string { return $this->preview->getEtag(); } - /** - * @inheritDoc - */ + #[Override] public function getMTime(): int { return $this->preview->getMtime(); } - /** - * @inheritDoc - */ + #[Override] public function getContent(): string { $stream = $this->storage->readPreview($this->preview); return stream_get_contents($stream); } - /** - * @inheritDoc - */ + #[Override] public function putContent($data): void { } - /** - * @inheritDoc - */ + #[Override] public function delete(): void { $this->storage->deletePreview($this->preview); $this->previewMapper->delete($this->preview); } - /** - * @inheritDoc - */ + #[Override] public function getMimeType(): string { return $this->preview->getMimetypeValue(); } - /** - * @inheritDoc - */ + #[Override] public function getExtension(): string { return $this->preview->getExtension(); } - /** - * @inheritDoc - */ + #[Override] public function read() { return $this->storage->readPreview($this->preview); } - /** - * @inheritDoc - */ + #[Override] public function write() { return false; } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3438534d07ed2..5d1d130b300bc 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -16,6 +16,7 @@ use OC\Preview\Db\PreviewMapper; use OCP\IConfig; use OCP\Server; +use Override; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -27,14 +28,17 @@ public function __construct( ) { } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { return $this->getBackend()->writePreview($preview, $stream); } + #[Override] public function readPreview(Preview $preview): mixed { return $this->getBackend()->readPreview($preview); } + #[Override] public function deletePreview(Preview $preview): void { $this->getBackend()->deletePreview($preview); } @@ -53,10 +57,12 @@ private function getBackend(): IPreviewStorage { return $this->backend; } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + #[Override] public function scan(): int { return $this->getBackend()->scan(); } From 66f50bd585f428862849db0db2847287e59453c9 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 30 Sep 2025 13:44:34 +0200 Subject: [PATCH 12/14] refactor(preview): Use same mimetype ids as filecache Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 13 ++- core/Command/Preview/ResetRenderedTexts.php | 2 +- lib/private/Files/Cache/LocalRootScanner.php | 14 +++ lib/private/Files/Cache/Scanner.php | 22 ++-- lib/private/Preview/Db/Preview.php | 73 ++++++------- lib/private/Preview/Db/PreviewMapper.php | 22 +--- lib/private/Preview/Generator.php | 102 ++++++------------ .../Preview/Storage/LocalPreviewStorage.php | 28 ++++- lib/private/Preview/Storage/PreviewFile.php | 8 +- .../Preview/Storage/StorageFactory.php | 6 +- lib/public/IPreview.php | 5 - tests/lib/Preview/GeneratorTest.php | 39 ++++--- tests/lib/Preview/MovePreviewJobTest.php | 23 +++- tests/lib/Preview/PreviewMapperTest.php | 5 +- tests/lib/Preview/PreviewServiceTest.php | 2 +- 15 files changed, 184 insertions(+), 180 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 0eaf409e433f4..ff13a6f37e22e 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -19,12 +19,15 @@ use OCP\DB\Exception; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; use Override; +use Psr\Log\LoggerInterface; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -36,6 +39,9 @@ public function __construct( private readonly StorageFactory $storageFactory, private readonly IDBConnection $connection, private readonly IRootFolder $rootFolder, + private readonly IMimeTypeDetector $mimeTypeDetector, + private readonly IMimeTypeLoader $mimeTypeLoader, + private readonly LoggerInterface $logger, IAppDataFactory $appDataFactory, ) { parent::__construct($time); @@ -125,8 +131,13 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { + $path = $fileId . '/' . $previewFile->getName(); /** @var SimpleFile $previewFile */ - $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview = Preview::fromPath($path, $this->mimeTypeDetector, $this->mimeTypeLoader); + if (!$preview) { + $this->logger->error('Unable to import old preview at path.'); + continue; + } $preview->setSize($previewFile->getSize()); $preview->setMtime($previewFile->getMtime()); $preview->setOldFileId($previewFile->getId()); diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index 6ec26ad6ac340..c4fcabbe5005b 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -93,7 +93,7 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; foreach ($this->getPreviewsToDelete() as $preview) { - $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); + $output->writeln('Deleting preview ' . $preview->getName($this->mimeTypeLoader) . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 3f4f70b865b01..79908d63fe220 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -8,7 +8,18 @@ */ namespace OC\Files\Cache; +use OCP\IConfig; +use OCP\Server; + class LocalRootScanner extends Scanner { + private string $previewFolder; + + public function __construct(\OC\Files\Storage\Storage $storage) { + parent::__construct($storage); + $config = Server::get(IConfig::class); + $this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview'; + } + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { if ($this->shouldScanPath($file)) { return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); @@ -27,6 +38,9 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc private function shouldScanPath(string $path): bool { $path = trim($path, '/'); + if (str_starts_with($path, $this->previewFolder)) { + return false; + } return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 75ca9da0abe15..2fa0dd09e4fd0 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -11,14 +11,15 @@ use OC\Files\Storage\Wrapper\Encryption; use OC\Files\Storage\Wrapper\Jail; use OC\Hooks\BasicEmitter; -use OC\SystemConfig; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; use OCP\Files\NotFoundException; use OCP\Files\Storage\ILockingStorage; use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use OCP\Server; use Psr\Log\LoggerInterface; /** @@ -65,19 +66,15 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; - private string $previewFolder; - public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); - /** @var SystemConfig $config */ - $config = \OC::$server->get(SystemConfig::class); - $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); - $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); - $this->lockingProvider = \OC::$server->get(ILockingProvider::class); - $this->connection = \OC::$server->get(IDBConnection::class); - $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; + $config = Server::get(IConfig::class); + $this->cacheActive = !$config->getSystemValueBool('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getSystemValueBool('filescanner_no_transactions', false); + $this->lockingProvider = Server::get(ILockingProvider::class); + $this->connection = Server::get(IDBConnection::class); } /** @@ -415,11 +412,6 @@ protected function scanChildren(string $path, $recursive, int $reuse, int $folde $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); - if (str_starts_with($path, $this->previewFolder)) { - // Preview scanning is handled in LocalPreviewStorage - return 0; - } - foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index fe30da6589c9b..7eb03e1e28992 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -12,7 +12,9 @@ use OCP\AppFramework\Db\Entity; use OCP\DB\Types; -use OCP\IPreview; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; +use OCP\Server; /** * Preview entity mapped to the oc_previews and oc_preview_locations table. @@ -91,42 +93,39 @@ public function __construct() { $this->addType('version', Types::BIGINT); } - public static function fromPath(string $path): Preview { + public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector, IMimeTypeLoader $mimeTypeLoader): Preview|false { $preview = new self(); $preview->setFileId((int)basename(dirname($path))); $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + $ok = preg_match('/(([0-9]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); - [0 => $baseName, 1 => $extension] = explode('.', $fileName); - $preview->setMimetype(match ($extension) { - 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, - 'png' => IPreview::MIMETYPE_PNG, - 'gif' => IPreview::MIMETYPE_GIF, - 'webp' => IPreview::MIMETYPE_WEBP, - default => IPreview::MIMETYPE_JPEG, - }); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $preview->setVersion(null); - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $preview->setVersion((int)$nameSplit[0]); + if ($ok !== 1) { + return false; } - $preview->setWidth((int)$nameSplit[$offset + 0]); - $preview->setHeight((int)$nameSplit[$offset + 1]); + [ + 2 => $version, + 3 => $width, + 4 => $height, + 6 => $crop, + 8 => $max, + ] = $matches; - $preview->setCropped(false); - $preview->setMax(false); - if (isset($nameSplit[$offset + 2])) { - $preview->setCropped($nameSplit[$offset + 2] === 'crop'); - $preview->setMax($nameSplit[$offset + 2] === 'max'); + $preview->setMimetype($mimeTypeLoader->getId($mimeTypeDetector->detectPath($fileName))); + + $preview->setWidth((int)$width); + $preview->setHeight((int)$height); + $preview->setCropped($crop === 'crop'); + $preview->setMax($max === 'max'); + + if (!empty($version)) { + $preview->setVersion((int)$version); } return $preview; } - public function getName(): string { + public function getName(IMimeTypeLoader $mimeTypeLoader): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { $path .= '-crop'; @@ -135,27 +134,23 @@ public function getName(): string { $path .= '-max'; } - $ext = $this->getExtension(); + $ext = $this->getExtension($mimeTypeLoader); $path .= '.' . $ext; return $path; } - public function getMimetypeValue(): string { - return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'image/jpeg', - IPreview::MIMETYPE_PNG => 'image/png', - IPreview::MIMETYPE_WEBP => 'image/webp', - IPreview::MIMETYPE_GIF => 'image/gif', + public function getExtension(IMimeTypeLoader $mimeTypeLoader): string { + return match ($this->getMimetypeValue($mimeTypeLoader)) { + 'image/png' => 'png', + 'image/gif' => 'gif', + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + default => 'png', }; } - public function getExtension(): string { - return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'jpg', - IPreview::MIMETYPE_PNG => 'png', - IPreview::MIMETYPE_WEBP => 'webp', - IPreview::MIMETYPE_GIF => 'gif', - }; + public function getMimetypeValue(IMimeTypeLoader $mimeTypeLoader): string { + return $mimeTypeLoader->getMimetypeById($this->mimetype) ?? 'image/jpeg'; } public function setBucketName(string $bucketName): void { diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index d4ef242eb21ff..7d399dc5f3dbf 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -13,6 +13,7 @@ use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; use OCP\IPreview; @@ -24,7 +25,9 @@ class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; private const LOCATION_TABLE_NAME = 'preview_locations'; - public function __construct(IDBConnection $db) { + public function __construct( + IDBConnection $db, + ) { parent::__construct($db, self::TABLE_NAME, Preview::class); } @@ -57,23 +60,6 @@ public function getAvailablePreviews(array $fileIds): array { return $previews; } - public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { - $selectQb = $this->db->getQueryBuilder(); - $this->joinLocation($selectQb) - ->where( - $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), - $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), - $selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)), - $selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)), - $selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)), - ); - try { - return $this->findEntity($selectQb); - } catch (DoesNotExistException) { - return null; - } - } - /** * @return \Generator */ diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 1743ca97e0d4f..664448b7d0197 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -152,7 +152,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); continue; } @@ -163,14 +163,14 @@ public function generatePreviews(File $file, array $specifications, ?string $mim && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { - $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); @@ -350,7 +350,21 @@ private function generateProviderPreview(File $file, int $width, int $height, bo } try { - return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); + $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setWidth($preview->width()); + $previewEntry->setHeight($preview->height()); + $previewEntry->setVersion($version); + $previewEntry->setMax($max); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); + $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + return $this->savePreview($previewEntry, $preview); } catch (NotPermittedException) { throw new NotFoundException(); } @@ -360,21 +374,6 @@ private function generateProviderPreview(File $file, int $width, int $height, bo throw new NotFoundException('No provider successfully handled the preview generation'); } - private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): string { - $path = ($version !== -1 ? $version . '-' : '') . $width . '-' . $height; - if ($crop) { - $path .= '-crop'; - } - if ($max) { - $path .= '-max'; - } - - $ext = $this->getExtension($mimeType); - $path .= '.' . $ext; - return $path; - } - - /** * @psalm-param IPreview::MODE_* $mode * @return int[] @@ -505,40 +504,6 @@ private function generatePreview( self::unguardWithSemaphore($sem); } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); - if ($cacheResult) { - $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); - return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); - } else { - return new InMemoryFile($path, $preview->data()); - } - } - - /** - * @throws \InvalidArgumentException - */ - private function getExtension(string $mimeType): string { - switch ($mimeType) { - case 'image/png': - return 'png'; - case 'image/jpeg': - return 'jpg'; - case 'image/webp': - return 'webp'; - case 'image/gif': - return 'gif'; - default: - throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); - } - } - - /** - * @throws InvalidPathException - * @throws NotFoundException - * @throws NotPermittedException - * @throws \OCP\DB\Exception - */ - public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); @@ -546,27 +511,28 @@ public function savePreview(File $file, int $width, int $height, bool $crop, boo $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); $previewEntry->setHeight($height); $previewEntry->setVersion($version); - $previewEntry->setMax($max); + $previewEntry->setMax(false); $previewEntry->setCropped($crop); $previewEntry->setEncrypted(false); - switch ($preview->dataMimeType()) { - case 'image/jpeg': - $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); - break; - case 'image/gif': - $previewEntry->setMimetype(IPreview::MIMETYPE_GIF); - break; - case 'image/webp': - $previewEntry->setMimetype(IPreview::MIMETYPE_WEBP); - break; - default: - $previewEntry->setMimetype(IPreview::MIMETYPE_PNG); - break; - } + $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); $previewEntry->setSize(0); + if ($cacheResult) { + $previewEntry = $this->savePreview($previewEntry, $preview); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + } else { + return new InMemoryFile($previewEntry->getName($this->mimeTypeLoader), $preview->data()); + } + } + /** + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException + * @throws \OCP\DB\Exception + */ + public function savePreview(Preview $previewEntry, IImage $preview): Preview { $previewEntry = $this->previewMapper->insert($previewEntry); // we need to save to DB first diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 8d85897115b3f..e522df7da8e7e 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -16,10 +16,13 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\DB\Exception; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; use Override; +use Psr\Log\LoggerInterface; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; @@ -33,6 +36,9 @@ public function __construct( private readonly StorageFactory $previewStorage, private readonly IAppConfig $appConfig, private readonly IDBConnection $connection, + private readonly IMimeTypeLoader $mimeTypeLoader, + private readonly IMimeTypeDetector $mimeTypeDetector, + private readonly LoggerInterface $logger, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -62,7 +68,7 @@ public function getPreviewRootFolder(): string { } private function constructPath(Preview $preview): string { - return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); } private function createParentFiles(string $path): bool { @@ -74,7 +80,7 @@ private function createParentFiles(string $path): bool { #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory - $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); if (!file_exists($sourcePath)) { return; } @@ -88,7 +94,7 @@ public function migratePreview(Preview $preview, SimpleFile $file): void { $this->createParentFiles($destinationPath); $ok = rename($sourcePath, $destinationPath); if (!$ok) { - throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); + throw new LogicException('Failed to move ' . $sourcePath . ' to ' . $destinationPath); } } @@ -100,7 +106,11 @@ public function scan(): int { $previewsFound = 0; foreach (new RecursiveIteratorIterator($scanner) as $file) { if ($file->isFile()) { - $preview = Preview::fromPath((string)$file); + $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector, $this->mimeTypeLoader); + if ($preview === false) { + $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); + continue; + } try { $preview->setSize($file->getSize()); $preview->setMtime($file->getMtime()); @@ -139,7 +149,15 @@ public function scan(): int { $this->previewMapper->insert($preview); // Move old flat preview to new format - $this->previewStorage->migratePreview($preview, $file); + $dirName = str_replace($this->getPreviewRootFolder(), '', $file->getPath()); + if (preg_match('/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9]+/', $dirName) !== 1) { + $previewPath = $this->constructPath($preview); + $this->createParentFiles($previewPath); + $ok = rename($file->getRealPath(), $previewPath); + if (!$ok) { + throw new LogicException('Failed to move ' . $file->getRealPath() . ' to ' . $previewPath); + } + } } catch (Exception $e) { if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { throw $e; diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index 8c9852673cce2..c7c7b59c97c3c 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -12,6 +12,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\IMimeTypeLoader; use OCP\Files\SimpleFS\ISimpleFile; use Override; @@ -20,12 +21,13 @@ public function __construct( private readonly Preview $preview, private readonly IPreviewStorage $storage, private readonly PreviewMapper $previewMapper, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { } #[Override] public function getName(): string { - return $this->preview->getName(); + return $this->preview->getName($this->mimeTypeLoader); } #[Override] @@ -61,12 +63,12 @@ public function delete(): void { #[Override] public function getMimeType(): string { - return $this->preview->getMimetypeValue(); + return $this->preview->getMimetypeValue($this->mimeTypeLoader); } #[Override] public function getExtension(): string { - return $this->preview->getExtension(); + return $this->preview->getExtension($this->mimeTypeLoader); } #[Override] diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 5d1d130b300bc..e33135be3ce6e 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -13,8 +13,6 @@ use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; -use OC\Preview\Db\PreviewMapper; -use OCP\IConfig; use OCP\Server; use Override; @@ -23,8 +21,6 @@ class StorageFactory implements IPreviewStorage { public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, - private readonly IConfig $config, - private readonly PreviewMapper $previewMapper, ) { } @@ -49,7 +45,7 @@ private function getBackend(): IPreviewStorage { } if ($this->objectStoreConfig->hasObjectStore()) { - $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); + $this->backend = Server::get(ObjectStorePreviewStorage::class); } else { $this->backend = Server::get(LocalPreviewStorage::class); } diff --git a/lib/public/IPreview.php b/lib/public/IPreview.php index cbd0e0ae525e4..3c9eadd45774a 100644 --- a/lib/public/IPreview.php +++ b/lib/public/IPreview.php @@ -29,11 +29,6 @@ interface IPreview { */ public const MODE_COVER = 'cover'; - public const MIMETYPE_JPEG = 0; - public const MIMETYPE_WEBP = 1; - public const MIMETYPE_PNG = 2; - public const MIMETYPE_GIF = 3; - /** * In order to improve lazy loading a closure can be registered which will be * called in case preview providers are actually requested diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 640c2ce8b36bf..fc48ebc181ab2 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -47,7 +47,12 @@ protected function setUp(): void { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); - $this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader->method('getId') + ->willReturnCallback(fn ($mimeType) => $mimeType === 'image/png' ? 42 : 43); + $this->mimeTypeLoader->method('getMimetypeById') + ->with(42) + ->willReturn('image/png'); $this->generator = new Generator( $this->config, @@ -57,7 +62,7 @@ protected function setUp(): void { $this->logger, $this->previewMapper, $this->storageFactory, - $this->mimetypeLoader, + $this->mimeTypeLoader, ); } @@ -91,7 +96,7 @@ public function testGetCachedPreview(): void { $maxPreview->setVersion(-1); $maxPreview->setCropped(false); $maxPreview->setStorageId(1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $previewFile = new Preview(); $previewFile->setWidth(256); @@ -101,7 +106,7 @@ public function testGetCachedPreview(): void { $previewFile->setVersion(-1); $previewFile->setCropped(false); $previewFile->setStorageId(1); - $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -130,7 +135,7 @@ public function testGetNewPreview(): void { ->with($this->equalTo([42])) ->willReturn([42 => []]); - $this->config->method('getSystemValue') + $this->config->method('getSystemValueString') ->willReturnCallback(function ($key, $default) { return $default; }); @@ -196,7 +201,7 @@ public function testGetNewPreview(): void { $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('insert') ->willReturnCallback(fn (Preview $preview): Preview => $preview); @@ -206,7 +211,7 @@ public function testGetNewPreview(): void { $this->storageFactory->method('writePreview') ->willReturnCallback(function (Preview $preview, string $data): int { - switch ($preview->getName()) { + switch ($preview->getName($this->mimeTypeLoader)) { case '2048-2048-max.png': $this->assertSame('my data', $data); return 1000; @@ -214,7 +219,7 @@ public function testGetNewPreview(): void { $this->assertSame('my resized data', $data); return 1000; } - $this->fail('file name is wrong:' . $preview->getName()); + $this->fail('file name is wrong:' . $preview->getName($this->mimeTypeLoader)); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -245,7 +250,7 @@ public function testInvalidMimeType(): void { $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype(42); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -269,7 +274,7 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $previewFile = new Preview(); $previewFile->setWidth(1024); @@ -278,7 +283,7 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $previewFile->setSize(1000); $previewFile->setCropped(true); $previewFile->setVersion(-1); - $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -316,7 +321,7 @@ public function testNoProvider(): void { $this->generator->getPreview($file, 100, 100); } - private function getMockImage($width, $height, $data = null) { + private function getMockImage(int $width, int $height, $data = null) { $image = $this->createMock(IImage::class); $image->method('height')->willReturn($width); $image->method('width')->willReturn($height); @@ -387,10 +392,10 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype(42); - $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); - $this->assertSame($maxPreview->getMimetypeValue(), 'image/png'); + $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimetypeValue($this->mimeTypeLoader), 'image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -410,7 +415,7 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $this->previewMapper->method('insert') ->willReturnCallback(function (Preview $preview) use ($filename): Preview { - $this->assertSame($preview->getName(), $filename); + $this->assertSame($preview->getName($this->mimeTypeLoader), $filename); return $preview; }); @@ -426,7 +431,7 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview->getName(), $result->getName()); + $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $result->getName()); } else { $this->assertSame($filename, $result->getName()); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index cd9f325365463..a5ac5ad51e3bf 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -17,12 +17,15 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\IAppConfig; use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\TestDox; use PHPUnit\Framework\MockObject\MockObject; +use Psr\Log\LoggerInterface; use Test\TestCase; /** @@ -35,6 +38,9 @@ class MovePreviewJobTest extends TestCase { private StorageFactory $storageFactory; private PreviewService $previewService; private IDBConnection $db; + private IMimeTypeLoader&MockObject $mimeTypeLoader; + private IMimeTypeDetector&MockObject $mimeTypeDetector; + private LoggerInterface&MockObject $logger; public function setUp(): void { parent::setUp(); @@ -75,6 +81,12 @@ public function setUp(): void { 'permissions' => $qb->createNamedParameter(0), 'checksum' => $qb->createNamedParameter('abcdefg'), ])->executeStatement(); + + $this->mimeTypeDetector = $this->createMock(IMimeTypeDetector::class); + $this->mimeTypeDetector->method('detectPath')->willReturn('image/png'); + $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader->method('getId')->with('image/png')->willReturn(42); + $this->logger = $this->createMock(LoggerInterface::class); } public function tearDown(): void { @@ -105,7 +117,10 @@ public function testMigrationLegacyPath(): void { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), - Server::get(IAppDataFactory::class) + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, + Server::get(IAppDataFactory::class), ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); @@ -133,6 +148,9 @@ public function testMigrationPath(): void { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); @@ -169,6 +187,9 @@ public function testMigrationPathWithVersion(): void { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index e4b2b1ab21ad4..19018c8b318c8 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -12,6 +12,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; use OCP\IPreview; use OCP\Server; @@ -23,10 +24,12 @@ class PreviewMapperTest extends TestCase { private PreviewMapper $previewMapper; private IDBConnection $connection; + private IMimeTypeLoader $mimeTypeLoader; public function setUp(): void { $this->previewMapper = Server::get(PreviewMapper::class); $this->connection = Server::get(IDBConnection::class); + $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); } public function testGetAvailablePreviews(): void { @@ -72,7 +75,7 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null): void $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); - $preview->setMimetype(IPreview::MIMETYPE_PNG); + $preview->setMimetype($this->mimeTypeLoader->getId('image/jpeg')); $preview->setEtag('abcdefg'); if ($locationId !== null) { diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index f9a8d88f664b2..fe8dd1c3d3342 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -46,7 +46,7 @@ public function testGetAvailableFileIds(): void { $preview->setSourceMimetype(1); $preview->setCropped(true); $preview->setEncrypted(false); - $preview->setMimetype(IPreview::MIMETYPE_JPEG); + $preview->setMimetype(42); $preview->setEtag('abc'); $preview->setMtime((new \DateTime())->getTimestamp()); $preview->setSize(0); From bef3996c3e21d401e11b3a5b2ea05be10189df2a Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 30 Sep 2025 15:56:31 +0200 Subject: [PATCH 13/14] fix(preview): Make version column a string And move it to a different table so that we don't have to pay the storage cost when not using it (most of the times). Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 137 +++++++++++------- core/Command/Preview/ResetRenderedTexts.php | 10 +- .../Version33000Date20250819110529.php | 16 +- lib/private/Files/Cache/LocalRootScanner.php | 14 +- lib/private/Preview/BackgroundCleanupJob.php | 2 +- lib/private/Preview/Db/Preview.php | 74 ++++++---- lib/private/Preview/Db/PreviewMapper.php | 75 +++++++++- lib/private/Preview/Generator.php | 35 ++--- lib/private/Preview/PreviewService.php | 6 +- .../Preview/Storage/IPreviewStorage.php | 20 ++- .../Preview/Storage/LocalPreviewStorage.php | 35 +++-- .../Storage/ObjectStorePreviewStorage.php | 29 ++-- lib/private/Preview/Storage/PreviewFile.php | 8 +- .../Preview/Storage/StorageFactory.php | 2 +- lib/private/Preview/Watcher.php | 16 +- lib/private/PreviewManager.php | 2 - lib/private/Server.php | 1 + lib/public/Preview/IVersionedPreviewFile.php | 3 +- .../lib/Preview/BackgroundCleanupJobTest.php | 2 +- tests/lib/Preview/GeneratorTest.php | 114 ++++++++------- tests/lib/Preview/MovePreviewJobTest.php | 35 +++-- tests/lib/Preview/PreviewMapperTest.php | 8 +- tests/lib/Preview/PreviewServiceTest.php | 6 +- 23 files changed, 404 insertions(+), 246 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index ff13a6f37e22e..930f998a28bed 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -17,24 +17,26 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\Exception; +use OCP\DB\IResult; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; use OCP\Files\IMimeTypeDetector; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; -use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; +use OCP\IConfig; use OCP\IDBConnection; use Override; use Psr\Log\LoggerInterface; class MovePreviewJob extends TimedJob { private IAppData $appData; + private string $previewRootPath; public function __construct( ITimeFactory $time, private readonly IAppConfig $appConfig, + private readonly IConfig $config, private readonly PreviewMapper $previewMapper, private readonly StorageFactory $storageFactory, private readonly IDBConnection $connection, @@ -49,6 +51,7 @@ public function __construct( $this->appData = $appDataFactory->get('preview'); $this->setTimeSensitivity(self::TIME_INSENSITIVE); $this->setInterval(24 * 60 * 60); + $this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/'; } #[Override] @@ -57,49 +60,22 @@ protected function run(mixed $argument): void { return; } - $emptyHierarchicalPreviewFolders = false; - $startTime = time(); while (true) { - // Check new hierarchical preview folders first - if (!$emptyHierarchicalPreviewFolders) { - $qb = $this->connection->getQueryBuilder(); - $qb->select('*') - ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) - ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) - ->setMaxResults(100); - - $result = $qb->executeQuery(); - while ($row = $result->fetch()) { - $pathSplit = explode('/', $row['path']); - assert(count($pathSplit) >= 2); - $fileId = $pathSplit[count($pathSplit) - 2]; - $this->processPreviews($fileId, false); - } - } - - // And then the flat preview folder (legacy) - $emptyHierarchicalPreviewFolders = true; $qb = $this->connection->getQueryBuilder(); - $qb->select('*') + $qb->select('path') ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) + // Hierarchical preview folder structure + ->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%'))) + // Legacy flat preview folder structure + ->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%'))) ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); - $foundOldPreview = false; - while ($row = $result->fetch()) { - $pathSplit = explode('/', $row['path']); - assert(count($pathSplit) >= 2); - $fileId = $pathSplit[count($pathSplit) - 2]; - array_pop($pathSplit); - $this->processPreviews($fileId, true); - $foundOldPreview = true; - } + $foundPreviews = $this->processQueryResult($result); - if (!$foundOldPreview) { + if (!$foundPreviews) { break; } @@ -109,20 +85,46 @@ protected function run(mixed $argument): void { } } - try { - // Delete any leftover preview directory - $this->appData->getFolder('.')->delete(); - } catch (NotFoundException) { - // ignore - } $this->appConfig->setValueBool('core', 'previewMovedDone', true); } + private function processQueryResult(IResult $result): bool { + $foundPreview = false; + $fileIds = []; + $flatFileIds = []; + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = (int)$pathSplit[count($pathSplit) - 2]; + if (count($pathSplit) === 11) { + // Hierarchical structure + if (!in_array($fileId, $fileIds)) { + $fileIds[] = $fileId; + } + } else { + // Flat structure + if (!in_array($fileId, $flatFileIds)) { + $flatFileIds[] = $fileId; + } + } + $foundPreview = true; + } + + foreach ($fileIds as $fileId) { + $this->processPreviews($fileId, flatPath: false); + } + + foreach ($flatFileIds as $fileId) { + $this->processPreviews($fileId, flatPath: true); + } + return $foundPreview; + } + /** * @param array $previewFolders */ - private function processPreviews(int|string $fileId, bool $simplePaths): void { - $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + private function processPreviews(int $fileId, bool $flatPath): void { + $internalPath = $this->getInternalFolder((string)$fileId, $flatPath); $folder = $this->appData->getFolder($internalPath); /** @@ -133,7 +135,7 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { foreach ($folder->getDirectoryListing() as $previewFile) { $path = $fileId . '/' . $previewFile->getName(); /** @var SimpleFile $previewFile */ - $preview = Preview::fromPath($path, $this->mimeTypeDetector, $this->mimeTypeLoader); + $preview = Preview::fromPath($path, $this->mimeTypeDetector); if (!$preview) { $this->logger->error('Unable to import old preview at path.'); continue; @@ -160,23 +162,30 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { + /** @var Preview $preview */ $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setSourceMimetype($result[0]['mimetype']); + $preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result[0]['mimetype'])); try { $preview = $this->previewMapper->insert($preview); - } catch (Exception $e) { + } catch (Exception) { // We already have this preview in the preview table, skip + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); continue; } try { $this->storageFactory->migratePreview($preview, $file); + $qb = $this->connection->getQueryBuilder(); $qb->delete('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->executeStatement(); // Do not call $file->delete() as this will also delete the file from the file system } catch (\Exception $e) { @@ -184,35 +193,51 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void { throw $e; } } + } else { + // No matching fileId, delete preview + try { + $this->connection->beginTransaction(); + foreach ($previewFiles as $previewFile) { + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $file->delete(); + } + $this->connection->commit(); + } catch (Exception) { + $this->connection->rollback(); + } } - $this->deleteFolder($internalPath, $folder); + $this->deleteFolder($internalPath); } - public static function getInternalFolder(string $name, bool $simplePaths): string { - if ($simplePaths) { - return '/' . $name; + public static function getInternalFolder(string $name, bool $flatPath): string { + if ($flatPath) { + return $name; } return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; } - private function deleteFolder(string $path, ISimpleFolder $folder): void { - $folder->delete(); - + private function deleteFolder(string $path): void { $current = $path; while (true) { + $appDataPath = $this->previewRootPath . $current; + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($appDataPath)))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); + $current = dirname($current); if ($current === '/' || $current === '.' || $current === '') { break; } - $folder = $this->appData->getFolder($current); if (count($folder->getDirectoryListing()) !== 0) { break; } - $folder->delete(); } } } diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index c4fcabbe5005b..978128a609fd6 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -10,7 +10,6 @@ use OC\Preview\Db\Preview; use OC\Preview\PreviewService; -use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IAvatarManager; @@ -28,7 +27,6 @@ public function __construct( protected readonly IUserManager $userManager, protected readonly IAvatarManager $avatarManager, private readonly PreviewService $previewService, - private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct(); } @@ -93,7 +91,7 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; foreach ($this->getPreviewsToDelete() as $preview) { - $output->writeln('Deleting preview ' . $preview->getName($this->mimeTypeLoader) . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); + $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -112,9 +110,9 @@ private function deletePreviews(OutputInterface $output, bool $dryMode): void { */ private function getPreviewsToDelete(): \Generator { return $this->previewService->getPreviewsForMimeTypes([ - $this->mimeTypeLoader->getId('text/plain'), - $this->mimeTypeLoader->getId('text/markdown'), - $this->mimeTypeLoader->getId('text/x-markdown'), + 'text/plain', + 'text/markdown', + 'text/x-markdown' ]); } } diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 27bf9ab89b766..32a25b0e33c47 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -37,6 +37,14 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->setPrimaryKey(['id']); } + if (!$schema->hasTable('preview_versions')) { + $table = $schema->createTable('preview_versions'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('version', Types::STRING, ['notnull' => true, 'default' => '', 'length' => 1024]); + $table->setPrimaryKey(['id']); + } + if (!$schema->hasTable('previews')) { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); @@ -46,18 +54,18 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); - $table->addColumn('source_mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('mimetype_id', Types::INTEGER, ['notnull' => true]); + $table->addColumn('source_mimetype_id', Types::INTEGER, ['notnull' => true]); $table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work + $table->addColumn('version_id', Types::BIGINT, ['notnull' => true, 'default' => -1]); $table->setPrimaryKey(['id']); $table->addIndex(['file_id']); - $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'cropped', 'version'], 'previews_file_uniq_idx'); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype_id', 'cropped', 'version_id'], 'previews_file_uniq_idx'); } return $schema; diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 79908d63fe220..d5f7d40e1b650 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -10,6 +10,7 @@ use OCP\IConfig; use OCP\Server; +use Override; class LocalRootScanner extends Scanner { private string $previewFolder; @@ -20,6 +21,7 @@ public function __construct(\OC\Files\Storage\Storage $storage) { $this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview'; } + #[Override] public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { if ($this->shouldScanPath($file)) { return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); @@ -28,6 +30,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = } } + #[Override] public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { if ($this->shouldScanPath($path)) { return parent::scan($path, $recursive, $reuse, $lock); @@ -36,11 +39,16 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc } } - private function shouldScanPath(string $path): bool { - $path = trim($path, '/'); + #[Override] + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { if (str_starts_with($path, $this->previewFolder)) { - return false; + return 0; } + return parent::scanChildren($path, $recursive, $reuse, $folderId, $lock, $oldSize, $etagChanged); + } + + private function shouldScanPath(string $path): bool { + $path = trim($path, '/'); return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 257bebb1d5ded..f6122fd0e1255 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -35,7 +35,7 @@ public function __construct( public function run($argument): void { foreach ($this->getDeletedFiles() as $fileId) { $previewIds = []; - foreach ($this->previewService->getAvailablePreviewForFile($fileId) as $preview) { + foreach ($this->previewService->getAvailablePreviewsForFile($fileId) as $preview) { $this->previewService->deletePreview($preview); } } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index 7eb03e1e28992..d3b3ab5ad07b2 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -13,8 +13,6 @@ use OCP\AppFramework\Db\Entity; use OCP\DB\Types; use OCP\Files\IMimeTypeDetector; -use OCP\Files\IMimeTypeLoader; -use OCP\Server; /** * Preview entity mapped to the oc_previews and oc_preview_locations table. @@ -35,10 +33,10 @@ * @method void setHeight(int $height) * @method bool isCropped() Get whether the preview is cropped or not. * @method void setCropped(bool $cropped) - * @method void setMimetype(int $mimetype) Set the mimetype of the preview. - * @method int getMimetype() Get the mimetype of the preview. - * @method void setSourceMimetype(int $sourceMimetype) Set the mimetype of the source file. - * @method int getSourceMimetype() Get the mimetype of the source file. + * @method void setMimetypeId(int $mimetype) Set the mimetype of the preview. + * @method int getMimetypeId() Get the mimetype of the preview. + * @method void setSourceMimetypeId(int $sourceMimetype) Set the mimetype of the source file. + * @method int getSourceMimetypeId() Get the mimetype of the source file. * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) * @method int getSize() Get the size of the preview. @@ -47,8 +45,8 @@ * @method void setMax(bool $max) * @method string getEtag() Get the etag of the preview. * @method void setEtag(string $etag) - * @method int|null getVersion() Get the version for files_versions_s3 - * @method void setVersion(?int $version) + * @method string|null getVersion() Get the version for files_versions_s3 + * @method void setVersionId(int $versionId) * @method bool|null getIs() Get the version for files_versions_s3 * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted. * @method void setEncrypted(bool $encrypted) @@ -64,15 +62,17 @@ class Preview extends Entity { protected ?string $objectStoreName = null; protected ?int $width = null; protected ?int $height = null; - protected ?int $mimetype = null; - - protected ?int $sourceMimetype = null; + protected ?int $mimetypeId = null; + protected ?int $sourceMimetypeId = null; + protected string $mimetype = 'application/octet-stream'; + protected string $sourceMimetype = 'application/octet-stream'; protected ?int $mtime = null; protected ?int $size = null; protected ?bool $max = null; protected ?bool $cropped = null; protected ?string $etag = null; - protected ?int $version = null; + protected ?string $version = null; + protected ?int $versionId = null; protected ?bool $encrypted = null; public function __construct() { @@ -82,23 +82,23 @@ public function __construct() { $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); - $this->addType('mimetype', Types::INTEGER); - $this->addType('sourceMimetype', Types::INTEGER); + $this->addType('mimetypeId', Types::INTEGER); + $this->addType('sourceMimetypeId', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); $this->addType('max', Types::BOOLEAN); $this->addType('cropped', Types::BOOLEAN); $this->addType('encrypted', Types::BOOLEAN); $this->addType('etag', Types::STRING); - $this->addType('version', Types::BIGINT); + $this->addType('versionId', Types::STRING); } - public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector, IMimeTypeLoader $mimeTypeLoader): Preview|false { + public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector): Preview|false { $preview = new self(); $preview->setFileId((int)basename(dirname($path))); $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); - $ok = preg_match('/(([0-9]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); + $ok = preg_match('/(([A-Za-z0-9\+\/]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); if ($ok !== 1) { return false; @@ -108,11 +108,11 @@ public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetecto 2 => $version, 3 => $width, 4 => $height, - 6 => $crop, - 8 => $max, + 6 => $max, + 8 => $crop, ] = $matches; - $preview->setMimetype($mimeTypeLoader->getId($mimeTypeDetector->detectPath($fileName))); + $preview->setMimeType($mimeTypeDetector->detectPath($fileName)); $preview->setWidth((int)$width); $preview->setHeight((int)$height); @@ -120,12 +120,12 @@ public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetecto $preview->setMax($max === 'max'); if (!empty($version)) { - $preview->setVersion((int)$version); + $preview->setVersion($version); } return $preview; } - public function getName(IMimeTypeLoader $mimeTypeLoader): string { + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { $path .= '-crop'; @@ -134,13 +134,13 @@ public function getName(IMimeTypeLoader $mimeTypeLoader): string { $path .= '-max'; } - $ext = $this->getExtension($mimeTypeLoader); + $ext = $this->getExtension(); $path .= '.' . $ext; return $path; } - public function getExtension(IMimeTypeLoader $mimeTypeLoader): string { - return match ($this->getMimetypeValue($mimeTypeLoader)) { + public function getExtension(): string { + return match ($this->getMimeType()) { 'image/png' => 'png', 'image/gif' => 'gif', 'image/jpeg' => 'jpg', @@ -149,10 +149,6 @@ public function getExtension(IMimeTypeLoader $mimeTypeLoader): string { }; } - public function getMimetypeValue(IMimeTypeLoader $mimeTypeLoader): string { - return $mimeTypeLoader->getMimetypeById($this->mimetype) ?? 'image/jpeg'; - } - public function setBucketName(string $bucketName): void { $this->bucketName = $bucketName; } @@ -160,4 +156,24 @@ public function setBucketName(string $bucketName): void { public function setObjectStoreName(string $objectStoreName): void { $this->objectStoreName = $objectStoreName; } + + public function setVersion(?string $version): void { + $this->version = $version; + } + + public function getMimeType(): string { + return $this->mimetype; + } + + public function setMimeType(string $mimeType): void { + $this->mimetype = $mimeType; + } + + public function getSourceMimeType(): string { + return $this->sourceMimetype; + } + + public function setSourceMimeType(string $mimeType): void { + $this->sourceMimetype = $mimeType; + } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 7d399dc5f3dbf..e6ca2e720f358 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,13 +9,13 @@ namespace OC\Preview\Db; -use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; -use OCP\IPreview; +use Override; /** * @template-extends QBMapper @@ -24,18 +24,74 @@ class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; private const LOCATION_TABLE_NAME = 'preview_locations'; + private const VERSION_TABLE_NAME = 'preview_versions'; public function __construct( IDBConnection $db, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct($db, self::TABLE_NAME, Preview::class); } + protected function mapRowToEntity(array $row): Entity { + $row['mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['mimetype_id']); + $row['source_mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['source_mimetype_id']); + + return parent::mapRowToEntity($row); + } + + #[Override] + public function insert(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->insert(self::VERSION_TABLE_NAME) + ->values([ + 'version' => $preview->getVersion(), + 'file_id' => $preview->getFileId(), + ]) + ->executeStatement(); + $entity->setVersionId($qb->getLastInsertId()); + } + return parent::insert($preview); + } + + #[Override] + public function update(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + return parent::update($preview); + } + + #[Override] + public function delete(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::VERSION_TABLE_NAME) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($preview->getFileId()))) + ->andWhere($qb->expr()->eq('version', $qb->createNamedParameter($preview->getVersion()))) + ->executeStatement(); + } + + return parent::delete($entity); + } + /** * @return \Generator * @throws Exception */ - public function getAvailablePreviewForFile(int $fileId): \Generator { + public function getAvailablePreviewsForFile(int $fileId): \Generator { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) ->where($selectQb->expr()->eq('p.file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); @@ -82,10 +138,13 @@ public function deleteByIds(array $previewIds): void { } protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { - return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') + return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name', 'v.version') ->from(self::TABLE_NAME, 'p') - ->leftJoin('p', 'preview_locations', 'l', $qb->expr()->eq( + ->leftJoin('p', self::LOCATION_TABLE_NAME, 'l', $qb->expr()->eq( 'p.location_id', 'l.id' + )) + ->leftJoin('p', self::VERSION_TABLE_NAME, 'v', $qb->expr()->eq( + 'p.version_id', 'v.id' )); } @@ -127,15 +186,15 @@ public function getPreviews(int $lastId, int $limit = 1000): \Generator { } /** - * @param int[] $mimeTypes + * @param string[] $mimeTypes * @return \Generator */ public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { $qb = $this->db->getQueryBuilder(); $this->joinLocation($qb) ->where($qb->expr()->orX( - ...array_map(function (int $mimeType) use ($qb) { - return $qb->expr()->eq('source_mimetype', $qb->createNamedParameter($mimeType, IQueryBuilder::PARAM_INT)); + ...array_map(function (string $mimeType) use ($qb): string { + return $qb->expr()->eq('source_mimetype_id', $qb->createNamedParameter($this->mimeTypeLoader->getId($mimeType), IQueryBuilder::PARAM_INT)); }, $mimeTypes) )); return $this->yieldEntities($qb); diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 664448b7d0197..82c4ec88363f2 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,7 +12,6 @@ use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -39,7 +38,6 @@ public function __construct( private LoggerInterface $logger, private PreviewMapper $previewMapper, private StorageFactory $storageFactory, - private IMimeTypeLoader $mimeTypeLoader, ) { } @@ -111,9 +109,9 @@ public function generatePreviews(File $file, array $specifications, ?string $mim [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = -1; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = (int)$file->getPreviewVersion(); + $previewVersion = $file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that @@ -152,7 +150,7 @@ public function generatePreviews(File $file, array $specifications, ?string $mim // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); continue; } @@ -163,14 +161,14 @@ public function generatePreviews(File $file, array $specifications, ?string $mim && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { - $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader)); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); @@ -298,7 +296,7 @@ public function getNumConcurrentPreviews(string $type): int { * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(array $previews, File $file, string $mimeType, int $version): Preview { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?string $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { @@ -313,7 +311,7 @@ private function getMaxPreview(array $previews, File $file, string $mimeType, in return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): Preview { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?string $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -353,14 +351,14 @@ private function generateProviderPreview(File $file, int $width, int $height, bo $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); - $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setSourceMimeType($file->getMimeType()); $previewEntry->setWidth($preview->width()); $previewEntry->setHeight($preview->height()); $previewEntry->setVersion($version); $previewEntry->setMax($max); $previewEntry->setCropped($crop); $previewEntry->setEncrypted(false); - $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setMimetype($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); $previewEntry->setSize(0); @@ -468,7 +466,7 @@ private function generatePreview( bool $crop, int $maxWidth, int $maxHeight, - ?int $version, + ?string $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -508,21 +506,21 @@ private function generatePreview( $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); - $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setSourceMimeType($file->getMimeType()); $previewEntry->setHeight($height); $previewEntry->setVersion($version); $previewEntry->setMax(false); $previewEntry->setCropped($crop); $previewEntry->setEncrypted(false); - $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setMimeType($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); $previewEntry->setSize(0); if ($cacheResult) { $previewEntry = $this->savePreview($previewEntry, $preview); - return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); } else { - return new InMemoryFile($previewEntry->getName($this->mimeTypeLoader), $preview->data()); + return new InMemoryFile($previewEntry->getName(), $preview->data()); } } @@ -540,7 +538,10 @@ public function savePreview(Preview $previewEntry, IImage $preview): Preview { if ($preview instanceof IStreamImage) { $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); } else { - $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $preview->data()); + rewind($stream); + $size = $this->storageFactory->writePreview($previewEntry, $stream); } if (!$size) { throw new \RuntimeException('Unable to write preview file'); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php index 0e7c66dc12dd5..8d30ae8a40212 100644 --- a/lib/private/Preview/PreviewService.php +++ b/lib/private/Preview/PreviewService.php @@ -70,12 +70,12 @@ public function getAvailableFileIds(): \Generator { /** * @return \Generator */ - public function getAvailablePreviewForFile(int $fileId): \Generator { - return $this->previewMapper->getAvailablePreviewForFile($fileId); + public function getAvailablePreviewsForFile(int $fileId): \Generator { + return $this->previewMapper->getAvailablePreviewsForFile($fileId); } /** - * @param int[] $mimeTypes + * @param string[] $mimeTypes * @return \Generator */ public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 5646432683848..1d6b128f8f022 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -10,32 +10,44 @@ namespace OC\Preview\Storage; +use Exception; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; interface IPreviewStorage { /** - * @param resource|string $stream + * @param resource $stream * @throws NotPermittedException + * @throws NotFoundException */ - public function writePreview(Preview $preview, mixed $stream): false|int; + public function writePreview(Preview $preview, mixed $stream): int; /** * @param Preview $preview - * @return resource|false + * @return resource + * @throws NotPermittedException + * @throws NotFoundException */ public function readPreview(Preview $preview): mixed; + /** + * @throws NotPermittedException + */ public function deletePreview(Preview $preview): void; /** * Migration helper * * To remove at some point - * @throw \Exception + * @throws Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + /** + * @throws NotPermittedException + * @throws NotFoundException + */ public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index e522df7da8e7e..bd5e1a97818cd 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -17,7 +17,8 @@ use OC\Preview\Db\PreviewMapper; use OCP\DB\Exception; use OCP\Files\IMimeTypeDetector; -use OCP\Files\IMimeTypeLoader; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; @@ -33,10 +34,8 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, private readonly PreviewMapper $previewMapper, - private readonly StorageFactory $previewStorage, private readonly IAppConfig $appConfig, private readonly IDBConnection $connection, - private readonly IMimeTypeLoader $mimeTypeLoader, private readonly IMimeTypeDetector $mimeTypeDetector, private readonly LoggerInterface $logger, ) { @@ -45,22 +44,28 @@ public function __construct( } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): int { $previewPath = $this->constructPath($preview); - if (!$this->createParentFiles($previewPath)) { - return false; - } + $this->createParentFiles($previewPath); return file_put_contents($previewPath, $stream); } #[Override] public function readPreview(Preview $preview): mixed { - return @fopen($this->constructPath($preview), 'r'); + $previewPath = $this->constructPath($preview); + $resource = @fopen($previewPath, 'r'); + if ($resource === false) { + throw new NotFoundException('Unable to open preview stream at ' . $previewPath); + } + return $resource; } #[Override] public function deletePreview(Preview $preview): void { - @unlink($this->constructPath($preview)); + $previewPath = $this->constructPath($preview); + if (!@unlink($previewPath) && is_file($previewPath)) { + throw new NotPermittedException('Unable to delete preview at ' . $previewPath); + } } public function getPreviewRootFolder(): string { @@ -68,19 +73,21 @@ public function getPreviewRootFolder(): string { } private function constructPath(Preview $preview): string { - return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } - private function createParentFiles(string $path): bool { + private function createParentFiles(string $path): void { $dirname = dirname($path); @mkdir($dirname, recursive: true); - return is_dir($dirname); + if (!is_dir($dirname)) { + throw new NotPermittedException("Unable to create directory '$dirname'"); + } } #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory - $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); if (!file_exists($sourcePath)) { return; } @@ -106,7 +113,7 @@ public function scan(): int { $previewsFound = 0; foreach (new RecursiveIteratorIterator($scanner) as $file) { if ($file->isFile()) { - $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector, $this->mimeTypeLoader); + $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector); if ($preview === false) { $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); continue; diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 88470802667f7..3e4337fbf28ab 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -15,6 +15,7 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\NotPermittedException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use Override; @@ -41,15 +42,7 @@ public function __construct( } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { - if (!is_resource($stream)) { - $fh = fopen('php://temp', 'w+'); - fwrite($fh, $stream); - rewind($fh); - - $stream = $fh; - } - + public function writePreview(Preview $preview, mixed $stream): int { $size = 0; $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { $size = $writtenSize; @@ -61,7 +54,11 @@ public function writePreview(Preview $preview, mixed $stream): false|int { 'config' => $config, ] = $this->getObjectStoreForPreview($preview); - $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + try { + $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to save preview to object store', previous: $exception); + } return $size; } @@ -71,7 +68,11 @@ public function readPreview(Preview $preview): mixed { 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + try { + return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to read preview from object store', previous: $exception); + } } #[Override] @@ -80,7 +81,11 @@ public function deletePreview(Preview $preview): void { 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + try { + $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to delete preview from object store', previous: $exception); + } } #[Override] diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index c7c7b59c97c3c..64cafb454707a 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -12,7 +12,6 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\IMimeTypeLoader; use OCP\Files\SimpleFS\ISimpleFile; use Override; @@ -21,13 +20,12 @@ public function __construct( private readonly Preview $preview, private readonly IPreviewStorage $storage, private readonly PreviewMapper $previewMapper, - private readonly IMimeTypeLoader $mimeTypeLoader, ) { } #[Override] public function getName(): string { - return $this->preview->getName($this->mimeTypeLoader); + return $this->preview->getName(); } #[Override] @@ -63,12 +61,12 @@ public function delete(): void { #[Override] public function getMimeType(): string { - return $this->preview->getMimetypeValue($this->mimeTypeLoader); + return $this->preview->getMimetype(); } #[Override] public function getExtension(): string { - return $this->preview->getExtension($this->mimeTypeLoader); + return $this->preview->getExtension(); } #[Override] diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index e33135be3ce6e..b15031c6a12ed 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -25,7 +25,7 @@ public function __construct( } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): int { return $this->getBackend()->writePreview($preview, $stream); } diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index ea0f72796ae5b..9b95e87d2ed03 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -13,6 +13,7 @@ use OCP\Files\FileInfo; use OCP\Files\Folder; use OCP\Files\Node; +use OCP\IDBConnection; /** * Class Watcher @@ -26,8 +27,9 @@ class Watcher { * Watcher constructor. */ public function __construct( - readonly private StorageFactory $storageFactory, - readonly private PreviewMapper $previewMapper, + private readonly StorageFactory $storageFactory, + private readonly PreviewMapper $previewMapper, + private readonly IDBConnection $connection, ) { } @@ -47,8 +49,14 @@ protected function deleteNode(FileInfo $node): void { } [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$nodeId]); - foreach ($previews as $preview) { - $this->storageFactory->deletePreview($preview); + $this->connection->beginTransaction(); + try { + foreach ($previews as $preview) { + $this->storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + } finally { + $this->connection->commit(); } } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 480408d772436..fb88409287260 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,7 +16,6 @@ use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; @@ -140,7 +139,6 @@ private function getGenerator(): Generator { $this->container->get(LoggerInterface::class), $this->container->get(PreviewMapper::class), $this->container->get(StorageFactory::class), - $this->container->get(IMimeTypeLoader::class), ); } return $this->generator; diff --git a/lib/private/Server.php b/lib/private/Server.php index 974838443331c..927d2ce322422 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -308,6 +308,7 @@ public function __construct($webRoot, \OC\Config $config) { return new Watcher( $c->get(\OC\Preview\Storage\StorageFactory::class), $c->get(PreviewMapper::class), + $c->get(IDBConnection::class), ); }); diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 7d68fe8d15e87..842ae877339ea 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -18,8 +18,7 @@ */ interface IVersionedPreviewFile { /** - * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(); + public function getPreviewVersion(): string; } diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index a2c72cbad57eb..80df690ad767b 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -74,7 +74,7 @@ protected function tearDown(): void { $this->logout(); - foreach ($this->previewService->getAvailablePreviewForFile(5) as $preview) { + foreach ($this->previewService->getAvailablePreviewsForFile(5) as $preview) { $this->previewService->deletePreview($preview); } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index fc48ebc181ab2..ceaf483a6588c 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -14,7 +14,6 @@ use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\IConfig; @@ -22,10 +21,17 @@ use OCP\IPreview; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; +use OCP\Preview\IVersionedPreviewFile; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; use Test\TestCase; +abstract class VersionedPreviewFile implements IVersionedPreviewFile, File { + +} + class GeneratorTest extends TestCase { private IConfig&MockObject $config; private IPreview&MockObject $previewManager; @@ -35,7 +41,6 @@ class GeneratorTest extends TestCase { private LoggerInterface&MockObject $logger; private StorageFactory&MockObject $storageFactory; private PreviewMapper&MockObject $previewMapper; - private IMimeTypeLoader&MockObject $mimeTypeLoader; protected function setUp(): void { parent::setUp(); @@ -47,12 +52,6 @@ protected function setUp(): void { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); - $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); - $this->mimeTypeLoader->method('getId') - ->willReturnCallback(fn ($mimeType) => $mimeType === 'image/png' ? 42 : 43); - $this->mimeTypeLoader->method('getMimetypeById') - ->with(42) - ->willReturn('image/png'); $this->generator = new Generator( $this->config, @@ -62,14 +61,18 @@ protected function setUp(): void { $this->logger, $this->previewMapper, $this->storageFactory, - $this->mimeTypeLoader, ); } - private function getFile(int $fileId, string $mimeType): File { + private function getFile(int $fileId, string $mimeType, bool $hasVersion = false): File { $mountPoint = $this->createMock(IMountPoint::class); $mountPoint->method('getNumericStorageId')->willReturn(42); - $file = $this->createMock(File::class); + if ($hasVersion) { + $file = $this->createMock(VersionedPreviewFile::class); + $file->method('getPreviewVersion')->willReturn('abc'); + } else { + $file = $this->createMock(File::class); + } $file->method('isReadable') ->willReturn(true); $file->method('getMimeType') @@ -81,8 +84,10 @@ private function getFile(int $fileId, string $mimeType): File { return $file; } - public function testGetCachedPreview(): void { - $file = $this->getFile(42, 'myMimeType'); + #[TestWith([true])] + #[TestWith([false])] + public function testGetCachedPreview(bool $hasPreview): void { + $file = $this->getFile(42, 'myMimeType', $hasPreview); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) @@ -93,20 +98,20 @@ public function testGetCachedPreview(): void { $maxPreview->setHeight(1000); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); $maxPreview->setCropped(false); $maxPreview->setStorageId(1); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); + $maxPreview->setVersion($hasPreview ? 'abc' : null); + $maxPreview->setMimeType('image/png'); $previewFile = new Preview(); $previewFile->setWidth(256); $previewFile->setHeight(256); $previewFile->setMax(false); $previewFile->setSize(1000); - $previewFile->setVersion(-1); + $previewFile->setVersion($hasPreview ? 'abc' : null); $previewFile->setCropped(false); $previewFile->setStorageId(1); - $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); + $previewFile->setMimeType('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -120,12 +125,14 @@ public function testGetCachedPreview(): void { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame('256-256.png', $result->getName()); + $this->assertSame($hasPreview ? 'abc-256-256.png' : '256-256.png', $result->getName()); $this->assertSame(1000, $result->getSize()); } - public function testGetNewPreview(): void { - $file = $this->getFile(42, 'myMimeType'); + #[TestWith([true])] + #[TestWith([false])] + public function testGetNewPreview(bool $hasVersion): void { + $file = $this->getFile(42, 'myMimeType', $hasVersion); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) @@ -196,13 +203,6 @@ public function testGetNewPreview(): void { $image->method('data') ->willReturn('my data'); - $maxPreview = new Preview(); - $maxPreview->setWidth(2048); - $maxPreview->setHeight(2048); - $maxPreview->setMax(true); - $maxPreview->setSize(1000); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); - $this->previewMapper->method('insert') ->willReturnCallback(fn (Preview $preview): Preview => $preview); @@ -210,16 +210,28 @@ public function testGetNewPreview(): void { ->willReturnCallback(fn (Preview $preview): Preview => $preview); $this->storageFactory->method('writePreview') - ->willReturnCallback(function (Preview $preview, string $data): int { - switch ($preview->getName($this->mimeTypeLoader)) { - case '2048-2048-max.png': - $this->assertSame('my data', $data); - return 1000; - case '256-256.png': - $this->assertSame('my resized data', $data); - return 1000; + ->willReturnCallback(function (Preview $preview, mixed $data) use ($hasVersion): int { + $data = stream_get_contents($data); + if ($hasVersion) { + switch ($preview->getName()) { + case 'abc-2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case 'abc-256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + } else { + switch ($preview->getName()) { + case '2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case '256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } } - $this->fail('file name is wrong:' . $preview->getName($this->mimeTypeLoader)); + $this->fail('file name is wrong:' . $preview->getName()); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -231,7 +243,7 @@ public function testGetNewPreview(): void { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame('256-256.png', $result->getName()); + $this->assertSame($hasVersion ? 'abc-256-256.png' : '256-256.png', $result->getName()); $this->assertSame(1000, $result->getSize()); } @@ -249,8 +261,8 @@ public function testInvalidMimeType(): void { $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype(42); + $maxPreview->setVersion(null); + $maxPreview->setMimetype('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -273,8 +285,8 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); $previewFile = new Preview(); $previewFile->setWidth(1024); @@ -282,8 +294,8 @@ public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void $previewFile->setMax(false); $previewFile->setSize(1000); $previewFile->setCropped(true); - $previewFile->setVersion(-1); - $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); + $previewFile->setVersion(null); + $previewFile->setMimeType('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -321,7 +333,7 @@ public function testNoProvider(): void { $this->generator->getPreview($file, 100, 100); } - private function getMockImage(int $width, int $height, $data = null) { + private function getMockImage(int $width, int $height, string $data = '') { $image = $this->createMock(IImage::class); $image->method('height')->willReturn($width); $image->method('width')->willReturn($height); @@ -378,7 +390,7 @@ public static function dataSize(): array { ]; } - #[\PHPUnit\Framework\Attributes\DataProvider('dataSize')] + #[DataProvider('dataSize')] public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $crop, string $mode, int $expectedX, int $expectedY): void { $file = $this->getFile(42, 'myMimeType'); @@ -391,11 +403,11 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $maxPreview->setHeight($maxY); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype(42); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); - $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $maxX . '-' . $maxY . '-max.png'); - $this->assertSame($maxPreview->getMimetypeValue($this->mimeTypeLoader), 'image/png'); + $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimeType(), 'image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -415,7 +427,7 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $this->previewMapper->method('insert') ->willReturnCallback(function (Preview $preview) use ($filename): Preview { - $this->assertSame($preview->getName($this->mimeTypeLoader), $filename); + $this->assertSame($preview->getName(), $filename); return $preview; }); @@ -431,7 +443,7 @@ public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $result->getName()); + $this->assertSame($maxPreview->getName(), $result->getName()); } else { $this->assertSame($filename, $result->getName()); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index a5ac5ad51e3bf..8c9df4274f0f5 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -21,6 +21,7 @@ use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\IAppConfig; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\TestDox; @@ -35,6 +36,7 @@ class MovePreviewJobTest extends TestCase { private IAppData $previewAppData; private PreviewMapper $previewMapper; private IAppConfig&MockObject $appConfig; + private IConfig $config; private StorageFactory $storageFactory; private PreviewService $previewService; private IDBConnection $db; @@ -46,6 +48,7 @@ public function setUp(): void { parent::setUp(); $this->previewAppData = Server::get(IAppDataFactory::class)->get('preview'); $this->previewMapper = Server::get(PreviewMapper::class); + $this->config = Server::get(IConfig::class); $this->appConfig = $this->createMock(IAppConfig::class); $this->appConfig->expects($this->any()) ->method('getValueBool') @@ -71,7 +74,7 @@ public function setUp(): void { 'path_hash' => $qb->createNamedParameter(md5('test')), 'parent' => $qb->createNamedParameter(0), 'name' => $qb->createNamedParameter('abc'), - 'mimetype' => $qb->createNamedParameter(0), + 'mimetype' => $qb->createNamedParameter(42), 'size' => $qb->createNamedParameter(1000), 'mtime' => $qb->createNamedParameter(1000), 'storage_mtime' => $qb->createNamedParameter(1000), @@ -86,6 +89,7 @@ public function setUp(): void { $this->mimeTypeDetector->method('detectPath')->willReturn('image/png'); $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); $this->mimeTypeLoader->method('getId')->with('image/png')->willReturn(42); + $this->mimeTypeLoader->method('getMimetypeById')->with(42)->willReturn('image/png'); $this->logger = $this->createMock(LoggerInterface::class); } @@ -108,11 +112,12 @@ public function testMigrationLegacyPath(): void { $folder->newFile('128-128-crop.png', 'abcdefg'); $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); $this->assertEquals(2, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -124,7 +129,7 @@ public function testMigrationLegacyPath(): void { ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); } private static function getInternalFolder(string $name): string { @@ -139,11 +144,12 @@ public function testMigrationPath(): void { $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(2, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -155,7 +161,7 @@ public function testMigrationPath(): void { ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); } #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] @@ -178,11 +184,12 @@ public function testMigrationPathWithVersion(): void { $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(9, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -193,25 +200,25 @@ public function testMigrationPathWithVersion(): void { Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); - $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $previews = iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)); + $previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)); $this->assertEquals(9, count($previews)); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); $nameVersionMapping = []; foreach ($previews as $preview) { - $nameVersionMapping[$preview->getName()] = $preview->getVersion(); + $nameVersionMapping[$preview->getName($this->mimeTypeLoader)] = $preview->getVersion(); } $this->assertEquals([ - '1000-128-128.png' => 1000, '1000-128-128-crop.png' => 1000, + '1000-128-128.png' => 1000, '1000-256-256-max.png' => 1000, - '1001-128-128.png' => 1001, '1001-128-128-crop.png' => 1001, + '1001-128-128.png' => 1001, '1001-256-256-max.png' => 1001, - '128-128.png' => -1, - '128-128-crop.png' => -1, - '256-256-max.png' => -1, + '128-128-crop.png' => null, + '128-128.png' => null, + '256-256-max.png' => null, ], $nameVersionMapping); } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 19018c8b318c8..8e27a642473d0 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -12,9 +12,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; -use OCP\IPreview; use OCP\Server; use Test\TestCase; @@ -24,12 +22,10 @@ class PreviewMapperTest extends TestCase { private PreviewMapper $previewMapper; private IDBConnection $connection; - private IMimeTypeLoader $mimeTypeLoader; public function setUp(): void { $this->previewMapper = Server::get(PreviewMapper::class); $this->connection = Server::get(IDBConnection::class); - $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); } public function testGetAvailablePreviews(): void { @@ -71,11 +67,11 @@ private function createPreviewForFileId(int $fileId, ?int $bucket = null): void $preview->setCropped(true); $preview->setMax(true); $preview->setWidth(100); - $preview->setSourceMimetype(1); + $preview->setSourceMimeType('image/jpeg'); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); - $preview->setMimetype($this->mimeTypeLoader->getId('image/jpeg')); + $preview->setMimetype('image/jpeg'); $preview->setEtag('abcdefg'); if ($locationId !== null) { diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index fe8dd1c3d3342..f3f9c8ae895d3 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -13,7 +13,6 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OC\Preview\PreviewService; -use OCP\IPreview; use OCP\Server; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -24,6 +23,7 @@ #[CoversClass(PreviewService::class)] class PreviewServiceTest extends TestCase { private PreviewService $previewService; + private PreviewMapper $previewMapper; protected function setUp(): void { $this->previewService = Server::get(PreviewService::class); @@ -43,10 +43,10 @@ public function testGetAvailableFileIds(): void { $preview->setWidth($i); $preview->setHeight($i); $preview->setMax(true); - $preview->setSourceMimetype(1); + $preview->setSourceMimeType('image/jpeg'); $preview->setCropped(true); $preview->setEncrypted(false); - $preview->setMimetype(42); + $preview->setMimetype('image/jpeg'); $preview->setEtag('abc'); $preview->setMtime((new \DateTime())->getTimestamp()); $preview->setSize(0); From fed7a33d1f4567ed9fa4dc4ceb55af28d6e4ecf5 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Mon, 6 Oct 2025 17:49:05 +0200 Subject: [PATCH 14/14] refactor(preview-object-store): Refactor object store backend Simplify logic Signed-off-by: Carl Schwan --- .../Storage/ObjectStorePreviewStorage.php | 126 ++++++++++-------- 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 3e4337fbf28ab..0c94071d28e39 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -22,12 +22,12 @@ /** * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig - * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: ObjectStoreConfig} + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, urn: string} */ class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array> + * @var array> */ private array $objectStoreCache = []; @@ -49,13 +49,12 @@ public function writePreview(Preview $preview, mixed $stream): int { }); [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - 'config' => $config, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForNewPreview($preview); try { - $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + $store->writeObject($urn, $countStream); } catch (\Exception $exception) { throw new NotPermittedException('Unable to save preview to object store', previous: $exception); } @@ -65,24 +64,26 @@ public function writePreview(Preview $preview, mixed $stream): int { #[Override] public function readPreview(Preview $preview): mixed { [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForExistingPreview($preview); + try { - return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + return $store->readObject($urn); } catch (\Exception $exception) { - throw new NotPermittedException('Unable to read preview from object store', previous: $exception); + throw new NotPermittedException('Unable to read preview from object store with urn:' . $urn, previous: $exception); } } #[Override] public function deletePreview(Preview $preview): void { [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForExistingPreview($preview); + try { - $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + $store->deleteObject($urn); } catch (\Exception $exception) { throw new NotPermittedException('Unable to delete preview from object store', previous: $exception); } @@ -91,72 +92,83 @@ public function deletePreview(Preview $preview): void { #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore - $this->getObjectStoreForPreview($preview, true); + $this->getObjectStoreInfoForNewPreview($preview, migration: true); $this->previewMapper->update($preview); } /** * @return ObjectStoreDefinition */ - private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { - if ($preview->getObjectStoreName() === null) { - $config = $this->objectStoreConfig->getObjectStoreConfiguration($oldFallback ? 'root' : 'preview'); - $objectStoreName = $this->objectStoreConfig->resolveAlias($oldFallback ? 'root' : 'preview'); - - $bucketName = $config['arguments']['bucket']; - if ($config['arguments']['multibucket']) { - if ($this->isMultibucketPreviewDistributionEnabled) { - $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); - $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); - $bucketName .= '-preview-' . $bucketNumber; - } else { - $bucketName .= '0'; - } - } - $config['arguments']['bucket'] = $bucketName; + private function getObjectStoreInfoForExistingPreview(Preview $preview): array { + assert(!empty($preview->getObjectStoreName())); + assert(!empty($preview->getBucketName())); + + $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); + $config['arguments']['bucket'] = $preview->getBucketName(); + $objectStoreName = $preview->getObjectStoreName(); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } - $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); - $preview->setLocationId($locationId); - $preview->setObjectStoreName($objectStoreName); - $preview->setBucketName($bucketName); - } else { - $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); - $config['arguments']['bucket'] = $bucketName = $preview->getBucketName(); - $objectStoreName = $preview->getObjectStoreName(); + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreInfoForNewPreview(Preview $preview, bool $migration = false): array { + // When migrating old previews, use the 'root' object store configuration + $config = $this->objectStoreConfig->getObjectStoreConfiguration($migration ? 'root' : 'preview'); + $objectStoreName = $this->objectStoreConfig->resolveAlias($migration ? 'root' : 'preview'); + + $bucketName = $config['arguments']['bucket']; + if ($config['arguments']['multibucket']) { + if ($this->isMultibucketPreviewDistributionEnabled) { + // Spread the previews on different buckets depending on their corresponding fileId + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $bucketName .= '-preview-' . $bucketNumber; + } else { + // Put all previews in the root (0) bucket + $bucketName .= '0'; + } } + $config['arguments']['bucket'] = $bucketName; + + // Get the locationId corresponding to the bucketName and objectStoreName, this will create + // a new one, if no matching location is found in the DB. + $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); + $preview->setLocationId($locationId); + $preview->setObjectStoreName($objectStoreName); + $preview->setBucketName($bucketName); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } - $objectPrefix = $this->getObjectPrefix($preview, $config); + private function getObjectStore(string $objectStoreName, array $config): IObjectStore { + $bucketName = $config['arguments']['bucket']; if (!isset($this->objectStoreCache[$objectStoreName])) { $this->objectStoreCache[$objectStoreName] = []; - $this->objectStoreCache[$objectStoreName][$bucketName] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($config), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); } elseif (!isset($this->objectStoreCache[$objectStoreName][$bucketName])) { - $this->objectStoreCache[$objectStoreName][$bucketName] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($config), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); } return $this->objectStoreCache[$objectStoreName][$bucketName]; } - private function constructUrn(string $objectPrefix, int $id): string { - return $objectPrefix . $id; - } - - public function getObjectPrefix(Preview $preview, array $config): string { + public function getUrn(Preview $preview, array $config): string { if ($preview->getOldFileId()) { - return $config['arguments']['objectPrefix'] ?? 'uri:oid:'; + return ($config['arguments']['objectPrefix'] ?? 'urn:oid:') . $preview->getOldFileId(); } if (isset($config['arguments']['objectPrefix'])) { - return $config['arguments']['objectPrefix'] . 'preview:'; + return ($config['arguments']['objectPrefix'] . 'preview:') . $preview->getId(); } else { - return 'uri:oid:preview:'; + return 'uri:oid:preview:' . $preview->getId(); } }