diff --git a/3rdparty b/3rdparty index e6c45c6c0d4f9..c03621c8447f0 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit e6c45c6c0d4f92ffec621446fcc3b34584772b13 +Subproject commit c03621c8447f06d4a25e303f27016e54f329508c diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 709a4cd68ed8f..a7ed68633c34c 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -34,19 +34,18 @@ namespace OCA\DAV\Connector\Sabre; use OC\AppFramework\Http\Request; -use OC\Metadata\IMetadataManager; +use OC\FilesMetadata\Model\MetadataValueWrapper; use OCP\Constants; use OCP\Files\ForbiddenException; use OCP\Files\StorageNotAvailableException; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IConfig; use OCP\IPreview; use OCP\IRequest; use OCP\IUserSession; -use Psr\Log\LoggerInterface; use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\Exception\NotFound; use Sabre\DAV\IFile; -use Sabre\DAV\INode; use Sabre\DAV\PropFind; use Sabre\DAV\PropPatch; use Sabre\DAV\ServerPlugin; @@ -84,17 +83,7 @@ class FilesPlugin extends ServerPlugin { public const SHARE_NOTE = '{http://nextcloud.org/ns}note'; public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count'; public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count'; - public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size'; - public const FILE_METADATA_GPS = '{http://nextcloud.org/ns}file-metadata-gps'; - - public const ALL_METADATA_PROPS = [ - self::FILE_METADATA_SIZE => 'size', - self::FILE_METADATA_GPS => 'gps', - ]; - public const METADATA_MIMETYPES = [ - 'size' => 'image', - 'gps' => 'image', - ]; + public const FILE_METADATA_PREFIX = '{http://nextcloud.org/ns}metadata-'; /** Reference to main server object */ private ?Server $server = null; @@ -398,6 +387,10 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) $propFind->handle(self::DISPLAYNAME_PROPERTYNAME, function () use ($node) { return $node->getName(); }); + + foreach ($node->getFileInfo()->getMetadata() as $metadataKey => $metadataValue) { + $propFind->handle(self::FILE_METADATA_PREFIX.$metadataKey, $metadataValue->getValueAny()); + } } if ($node instanceof \OCA\DAV\Connector\Sabre\File) { @@ -427,31 +420,6 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) $propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) { return $node->getFileInfo()->getUploadTime(); }); - - if ($this->config->getSystemValueBool('enable_file_metadata', true)) { - foreach (self::ALL_METADATA_PROPS as $prop => $meta) { - $propFind->handle($prop, function () use ($node, $meta) { - if ($node->getFileInfo()->getMimePart() !== self::METADATA_MIMETYPES[$meta]) { - return []; - } - - if ($node->hasMetadata($meta)) { - $metadata = $node->getMetadata($meta); - } else { - // This code path should not be called since we try to preload - // the metadata when loading the folder or the search results - // in one go - $metadataManager = \OC::$server->get(IMetadataManager::class); - $metadata = $metadataManager->fetchMetadataFor($meta, [$node->getId()])[$node->getId()]; - - // TODO would be nice to display this in the profiler... - \OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata'); - } - - return $metadata->getValue(); - }); - } - } } if ($node instanceof Directory) { @@ -465,39 +433,6 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) $requestProperties = $propFind->getRequestedProperties(); - $requestedMetaData = []; - foreach ($requestProperties as $requestProperty) { - if (isset(self::ALL_METADATA_PROPS[$requestProperty])) { - $requestedMetaData[] = self::ALL_METADATA_PROPS[$requestProperty]; - } - } - if ( - $this->config->getSystemValueBool('enable_file_metadata', true) && - $propFind->getDepth() === 1 && - $requestedMetaData - ) { - $children = $node->getChildren(); - // Preloading of the metadata - - /** @var IMetaDataManager $metadataManager */ - $metadataManager = \OC::$server->get(IMetadataManager::class); - - foreach ($requestedMetaData as $requestedMeta) { - $relevantMimeType = self::METADATA_MIMETYPES[$requestedMeta]; - $childrenForMeta = array_filter($children, function (INode $child) use ($relevantMimeType) { - return $child instanceof File && $child->getFileInfo()->getMimePart() === $relevantMimeType; - }); - $fileIds = array_map(function (File $child) { - return $child->getFileInfo()->getId(); - }, $childrenForMeta); - $preloadedMetadata = $metadataManager->fetchMetadataFor($requestedMeta, $fileIds); - - foreach ($childrenForMeta as $child) { - $child->setMetadata($requestedMeta, $preloadedMetadata[$child->getFileInfo()->getId()]); - } - } - } - if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true) || in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) { $nbFiles = 0; @@ -583,6 +518,57 @@ public function handleUpdateProperties($path, PropPatch $propPatch) { $node->setCreationTime((int) $time); return true; }); + + + /** @var IFilesMetadataManager */ + $filesMetadataManager = \OCP\Server::get(IFilesMetadataManager::class); + $metadata = $filesMetadataManager->getMetadata((int)$node->getFileId()); + + foreach ($metadata->getKeys() as $metadataKey) { + $propPatch->handle(self::FILE_METADATA_PREFIX.$metadataKey, function (mixed $value) use ($metadata, $metadataKey, $filesMetadataManager) { + switch ($metadata->getType($metadataKey)) { + case MetadataValueWrapper::TYPE_STRING: + $metadata->set($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_INT: + $metadata->setInt($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_FLOAT: + $metadata->setFloat($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_BOOL: + $metadata->setBool($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_ARRAY: + $metadata->setArray($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_STRING_LIST: + $metadata->setStringList($metadataKey, $value); + break; + case MetadataValueWrapper::TYPE_INT_LIST: + $metadata->setIntList($metadataKey, $value); + break; + } + + $filesMetadataManager->saveMetadata($metadata); + return true; + }); + } + + foreach ($propPatch->getRemainingMutations() as $mutation) { + if (!str_starts_with($mutation, self::FILE_METADATA_PREFIX)) { + continue; + } + + $propPatch->handle($mutation, function ($value) use ($metadata, $mutation, $filesMetadataManager) { + $metadataKey = substr($mutation, strlen(self::FILE_METADATA_PREFIX)); + $metadata->set($metadataKey, $value); + $filesMetadataManager->saveMetadata($metadata); + return true; + }); + } + + /** * Disable modification of the displayname property for files and * folders via PROPPATCH. See PROPFIND for more information. diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index f5ac362719602..d55b002beb19a 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -37,17 +37,19 @@ use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OC\FilesMetadata\FilesMetadataManager; +use OC\ForbiddenException; +use OC\Metadata\MetadataManager; +use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\FileCacheUpdated; use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\NodeRemovedFromCache; use OCP\Files\File; -use OC\ForbiddenException; -use OC\Metadata\MetadataManager; -use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\StorageNotAvailableException; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IUserManager; use Psr\Log\LoggerInterface; use Symfony\Component\Console\Helper\Table; @@ -69,6 +71,7 @@ public function __construct( private IUserManager $userManager, private IRootFolder $rootFolder, private MetadataManager $metadataManager, + private FilesMetadataManager $filesMetadataManager, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, ) { @@ -137,9 +140,10 @@ protected function scanFiles(string $user, string $path, bool $scanMetadata, Out $this->abortIfInterrupted(); if ($scanMetadata) { $node = $this->rootFolder->get($path); - if ($node instanceof File) { - $this->metadataManager->generateMetadata($node, false); - } + $this->filesMetadataManager->refreshMetadata( + $node, + IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND + ); } }); diff --git a/apps/files_trashbin/lib/Trash/TrashItem.php b/apps/files_trashbin/lib/Trash/TrashItem.php index 3bfc905d3a163..119cb9a82e41a 100644 --- a/apps/files_trashbin/lib/Trash/TrashItem.php +++ b/apps/files_trashbin/lib/Trash/TrashItem.php @@ -190,4 +190,8 @@ public function getUploadTime(): int { public function getParentId(): int { return $this->fileInfo->getParentId(); } + + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } } diff --git a/core/Application.php b/core/Application.php index ca9b6ce2d8c1b..b42727ff90dc5 100644 --- a/core/Application.php +++ b/core/Application.php @@ -44,7 +44,9 @@ use OC\Authentication\Notifications\Notifier as AuthenticationNotifier; use OC\Core\Listener\BeforeTemplateRenderedListener; use OC\Core\Notification\CoreNotifier; -use OC\Metadata\FileEventListener; +use OC\FilesMetadata\Provider\ExifMetadataProvider; +use OC\FilesMetadata\Provider\OriginalDateTimeMetadataProvider; +use OC\FilesMetadata\Provider\SizeMetadataProvider; use OC\TagManager; use OCP\AppFramework\App; use OCP\AppFramework\Http\Events\BeforeLoginTemplateRenderedEvent; @@ -54,13 +56,10 @@ use OCP\DB\Events\AddMissingPrimaryKeyEvent; use OCP\DB\Types; use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Events\Node\NodeDeletedEvent; -use OCP\Files\Events\Node\NodeWrittenEvent; -use OCP\Files\Events\NodeRemovedFromCache; +use OCP\FilesMetadata\Event\MetadataLiveEvent; use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\UserDeletedEvent; use OCP\Util; -use OCP\IConfig; /** * Class Application @@ -332,16 +331,9 @@ public function __construct() { $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedWebAuthnCleanupListener::class); // Metadata - /** @var IConfig $config */ - $config = $container->get(IConfig::class); - if ($config->getSystemValueBool('enable_file_metadata', true)) { - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeDeletedEvent::class, FileEventListener::class); - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeRemovedFromCache::class, FileEventListener::class); - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeWrittenEvent::class, FileEventListener::class); - } + $eventDispatcher->addServiceListener(MetadataLiveEvent::class, ExifMetadataProvider::class, 1); + $eventDispatcher->addServiceListener(MetadataLiveEvent::class, SizeMetadataProvider::class); + $eventDispatcher->addServiceListener(MetadataLiveEvent::class, OriginalDateTimeMetadataProvider::class); // Tags $eventDispatcher->addServiceListener(UserDeletedEvent::class, TagManager::class); diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php new file mode 100644 index 0000000000000..66cca520bc25c --- /dev/null +++ b/core/Command/FilesMetadata/Get.php @@ -0,0 +1,97 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Command\FilesMetadata; + +use OC\DB\ConnectionAdapter; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\FilesMetadata\IFilesMetadataManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Get extends Command { + public function __construct( + private IRootFolder $rootFolder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct(); + } + + protected function configure() { + $this->setName('metadata:get') + ->setDescription('get stored metadata about a file, by its id') + ->addArgument( + 'fileId', + InputArgument::REQUIRED, + 'id of the file document' + ) + ->addArgument( + 'userId', + InputArgument::OPTIONAL, + 'file owner' + ) + ->addOption( + 'refresh', + '', + InputOption::VALUE_NONE, + 'refresh metadata' + ) + ->addOption( + 'reset', + '', + InputOption::VALUE_NONE, + 'refresh metadata from scratch' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileId = (int)$input->getArgument('fileId'); + if ($input->getOption('refresh')) { + $node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId); + $file = $node[0]; + if (null === $file) { + throw new NotFoundException(); + } + + $metadata = $this->filesMetadataManager->refreshMetadata( + $file, + IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, + $input->getOption('reset') + ); + } else { + $metadata = $this->filesMetadataManager->getMetadata($fileId); + } + + $output->writeln(json_encode($metadata, JSON_PRETTY_PRINT)); + + return 0; + } +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php new file mode 100644 index 0000000000000..c6a24d770f096 --- /dev/null +++ b/core/Migrations/Version28000Date20231004103301.php @@ -0,0 +1,80 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version28000Date20231004103301 extends SimpleMigrationStep { + + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('files_metadata')) { + $table = $schema->createTable('files_metadata'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]); + $table->addColumn('json', Types::TEXT); + $table->addColumn('sync_token', Types::STRING, ['length' => 15]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id'], 'files_meta_fileid'); + } + + if (!$schema->hasTable('files_metadata_index')) { + $table = $schema->createTable('files_metadata_index'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]); + $table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]); + $table->addColumn('meta_value', Types::STRING, ['notnull' => false, 'length' => 63]); + $table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]); +// $table->addColumn('meta_value_float', Types::FLOAT, ['notnull' => false, 'length' => 11,5]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['file_id', 'meta_key', 'meta_value'], 'f_meta_index'); + $table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i'); +// $table->addIndex(['file_id', 'meta_key', 'meta_value_float'], 'f_meta_index_f'); + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index d9e5dfcd775eb..d497a9581d1af 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -214,6 +214,8 @@ $application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager())); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class)); + + $application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class)); } else { $application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class)); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 4387852556d96..17b30c1b987f2 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -280,6 +280,14 @@ 'OCP\\Federation\\ICloudId' => $baseDir . '/lib/public/Federation/ICloudId.php', 'OCP\\Federation\\ICloudIdManager' => $baseDir . '/lib/public/Federation/ICloudIdManager.php', 'OCP\\Files' => $baseDir . '/lib/public/Files.php', + 'OCP\\FilesMetadata\\Event\\MetadataBackgroundEvent' => $baseDir . '/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataLiveEvent' => $baseDir . '/lib/public/FilesMetadata/Event/MetadataLiveEvent.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataNotFoundException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataTypeException' => $baseDir . '/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php', + 'OCP\\FilesMetadata\\IFilesMetadataManager' => $baseDir . '/lib/public/FilesMetadata/IFilesMetadataManager.php', + 'OCP\\FilesMetadata\\Model\\IFilesMetadata' => $baseDir . '/lib/public/FilesMetadata/Model/IFilesMetadata.php', + 'OCP\\FilesMetadata\\Model\\IMetadataQuery' => $baseDir . '/lib/public/FilesMetadata/Model/IMetadataQuery.php', 'OCP\\Files\\AlreadyExistsException' => $baseDir . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => $baseDir . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => $baseDir . '/lib/public/Files/Cache/AbstractCacheEvent.php', @@ -1005,6 +1013,7 @@ 'OC\\Core\\Command\\Encryption\\SetDefaultModule' => $baseDir . '/core/Command/Encryption/SetDefaultModule.php', 'OC\\Core\\Command\\Encryption\\ShowKeyStorageRoot' => $baseDir . '/core/Command/Encryption/ShowKeyStorageRoot.php', 'OC\\Core\\Command\\Encryption\\Status' => $baseDir . '/core/Command/Encryption/Status.php', + 'OC\\Core\\Command\\FilesMetadata\\Get' => $baseDir . '/core/Command/FilesMetadata/Get.php', 'OC\\Core\\Command\\Group\\Add' => $baseDir . '/core/Command/Group/Add.php', 'OC\\Core\\Command\\Group\\AddUser' => $baseDir . '/core/Command/Group/AddUser.php', 'OC\\Core\\Command\\Group\\Delete' => $baseDir . '/core/Command/Group/Delete.php', @@ -1175,6 +1184,7 @@ 'OC\\Core\\Migrations\\Version28000Date20230616104802' => $baseDir . '/core/Migrations/Version28000Date20230616104802.php', 'OC\\Core\\Migrations\\Version28000Date20230728104802' => $baseDir . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => $baseDir . '/core/Migrations/Version28000Date20230803221055.php', + 'OC\\Core\\Migrations\\Version28000Date20231004103301' => $baseDir . '/core/Migrations/Version28000Date20231004103301.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php', @@ -1259,6 +1269,18 @@ 'OC\\Federation\\CloudFederationShare' => $baseDir . '/lib/private/Federation/CloudFederationShare.php', 'OC\\Federation\\CloudId' => $baseDir . '/lib/private/Federation/CloudId.php', 'OC\\Federation\\CloudIdManager' => $baseDir . '/lib/private/Federation/CloudIdManager.php', + 'OC\\FilesMetadata\\Event\\MetadataEventBase' => $baseDir . '/lib/private/FilesMetadata/Event/MetadataEventBase.php', + 'OC\\FilesMetadata\\FilesMetadataManager' => $baseDir . '/lib/private/FilesMetadata/FilesMetadataManager.php', + 'OC\\FilesMetadata\\Job\\UpdateSingleMetadata' => $baseDir . '/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php', + 'OC\\FilesMetadata\\Listener\\MetadataDelete' => $baseDir . '/lib/private/FilesMetadata/Listener/MetadataDelete.php', + 'OC\\FilesMetadata\\Listener\\MetadataUpdate' => $baseDir . '/lib/private/FilesMetadata/Listener/MetadataUpdate.php', + 'OC\\FilesMetadata\\Model\\FilesMetadata' => $baseDir . '/lib/private/FilesMetadata/Model/FilesMetadata.php', + 'OC\\FilesMetadata\\Model\\MetadataQuery' => $baseDir . '/lib/private/FilesMetadata/Model/MetadataQuery.php', + 'OC\\FilesMetadata\\Model\\MetadataValueWrapper' => $baseDir . '/lib/private/FilesMetadata/Model/MetadataValueWrapper.php', + 'OC\\FilesMetadata\\Provider\\ExifMetadataProvider' => $baseDir . '/lib/private/FilesMetadata/Provider/ExifMetadataProvider.php', + 'OC\\FilesMetadata\\Provider\\OriginalDateTimeMetadataProvider' => $baseDir . '/lib/private/FilesMetadata/Provider/OriginalDateTimeMetadataProvider.php', + 'OC\\FilesMetadata\\Service\\IndexRequestService' => $baseDir . '/lib/private/FilesMetadata/Service/IndexRequestService.php', + 'OC\\FilesMetadata\\Service\\MetadataRequestService' => $baseDir . '/lib/private/FilesMetadata/Service/MetadataRequestService.php', 'OC\\Files\\AppData\\AppData' => $baseDir . '/lib/private/Files/AppData/AppData.php', 'OC\\Files\\AppData\\Factory' => $baseDir . '/lib/private/Files/AppData/Factory.php', 'OC\\Files\\Cache\\Cache' => $baseDir . '/lib/private/Files/Cache/Cache.php', @@ -1458,7 +1480,6 @@ 'OC\\Metadata\\IMetadataManager' => $baseDir . '/lib/private/Metadata/IMetadataManager.php', 'OC\\Metadata\\IMetadataProvider' => $baseDir . '/lib/private/Metadata/IMetadataProvider.php', 'OC\\Metadata\\MetadataManager' => $baseDir . '/lib/private/Metadata/MetadataManager.php', - 'OC\\Metadata\\Provider\\ExifProvider' => $baseDir . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 9b33577d66a26..9dc9240ecbe98 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -313,6 +313,14 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Federation\\ICloudId' => __DIR__ . '/../../..' . '/lib/public/Federation/ICloudId.php', 'OCP\\Federation\\ICloudIdManager' => __DIR__ . '/../../..' . '/lib/public/Federation/ICloudIdManager.php', 'OCP\\Files' => __DIR__ . '/../../..' . '/lib/public/Files.php', + 'OCP\\FilesMetadata\\Event\\MetadataBackgroundEvent' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php', + 'OCP\\FilesMetadata\\Event\\MetadataLiveEvent' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Event/MetadataLiveEvent.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataNotFoundException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php', + 'OCP\\FilesMetadata\\Exceptions\\FilesMetadataTypeException' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php', + 'OCP\\FilesMetadata\\IFilesMetadataManager' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/IFilesMetadataManager.php', + 'OCP\\FilesMetadata\\Model\\IFilesMetadata' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Model/IFilesMetadata.php', + 'OCP\\FilesMetadata\\Model\\IMetadataQuery' => __DIR__ . '/../../..' . '/lib/public/FilesMetadata/Model/IMetadataQuery.php', 'OCP\\Files\\AlreadyExistsException' => __DIR__ . '/../../..' . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => __DIR__ . '/../../..' . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/AbstractCacheEvent.php', @@ -1038,6 +1046,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Encryption\\SetDefaultModule' => __DIR__ . '/../../..' . '/core/Command/Encryption/SetDefaultModule.php', 'OC\\Core\\Command\\Encryption\\ShowKeyStorageRoot' => __DIR__ . '/../../..' . '/core/Command/Encryption/ShowKeyStorageRoot.php', 'OC\\Core\\Command\\Encryption\\Status' => __DIR__ . '/../../..' . '/core/Command/Encryption/Status.php', + 'OC\\Core\\Command\\FilesMetadata\\Get' => __DIR__ . '/../../..' . '/core/Command/FilesMetadata/Get.php', 'OC\\Core\\Command\\Group\\Add' => __DIR__ . '/../../..' . '/core/Command/Group/Add.php', 'OC\\Core\\Command\\Group\\AddUser' => __DIR__ . '/../../..' . '/core/Command/Group/AddUser.php', 'OC\\Core\\Command\\Group\\Delete' => __DIR__ . '/../../..' . '/core/Command/Group/Delete.php', @@ -1208,6 +1217,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version28000Date20230616104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230616104802.php', 'OC\\Core\\Migrations\\Version28000Date20230728104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230803221055.php', + 'OC\\Core\\Migrations\\Version28000Date20231004103301' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20231004103301.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php', @@ -1292,6 +1302,18 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Federation\\CloudFederationShare' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudFederationShare.php', 'OC\\Federation\\CloudId' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudId.php', 'OC\\Federation\\CloudIdManager' => __DIR__ . '/../../..' . '/lib/private/Federation/CloudIdManager.php', + 'OC\\FilesMetadata\\Event\\MetadataEventBase' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Event/MetadataEventBase.php', + 'OC\\FilesMetadata\\FilesMetadataManager' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/FilesMetadataManager.php', + 'OC\\FilesMetadata\\Job\\UpdateSingleMetadata' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php', + 'OC\\FilesMetadata\\Listener\\MetadataDelete' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Listener/MetadataDelete.php', + 'OC\\FilesMetadata\\Listener\\MetadataUpdate' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Listener/MetadataUpdate.php', + 'OC\\FilesMetadata\\Model\\FilesMetadata' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/FilesMetadata.php', + 'OC\\FilesMetadata\\Model\\MetadataQuery' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/MetadataQuery.php', + 'OC\\FilesMetadata\\Model\\MetadataValueWrapper' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Model/MetadataValueWrapper.php', + 'OC\\FilesMetadata\\Provider\\ExifMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Provider/ExifMetadataProvider.php', + 'OC\\FilesMetadata\\Provider\\OriginalDateTimeMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Provider/OriginalDateTimeMetadataProvider.php', + 'OC\\FilesMetadata\\Service\\IndexRequestService' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Service/IndexRequestService.php', + 'OC\\FilesMetadata\\Service\\MetadataRequestService' => __DIR__ . '/../../..' . '/lib/private/FilesMetadata/Service/MetadataRequestService.php', 'OC\\Files\\AppData\\AppData' => __DIR__ . '/../../..' . '/lib/private/Files/AppData/AppData.php', 'OC\\Files\\AppData\\Factory' => __DIR__ . '/../../..' . '/lib/private/Files/AppData/Factory.php', 'OC\\Files\\Cache\\Cache' => __DIR__ . '/../../..' . '/lib/private/Files/Cache/Cache.php', @@ -1491,7 +1513,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Metadata\\IMetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataManager.php', 'OC\\Metadata\\IMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataProvider.php', 'OC\\Metadata\\MetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/MetadataManager.php', - 'OC\\Metadata\\Provider\\ExifProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 67d01bb699907..93bd41aacb909 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -59,6 +59,7 @@ use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use OCP\Util; use Psr\Log\LoggerInterface; @@ -132,7 +133,8 @@ protected function getQueryBuilder() { return new CacheQueryBuilder( $this->connection, \OC::$server->getSystemConfig(), - \OC::$server->get(LoggerInterface::class) + \OC::$server->get(LoggerInterface::class), + \OC::$server->get(IFilesMetadataManager::class), ); } @@ -154,6 +156,7 @@ public function getNumericStorageId() { public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); if (is_string($file) || $file == '') { // normalize file @@ -169,12 +172,16 @@ public function get($file) { $data = $result->fetch(); $result->closeCursor(); + // @Louis: use asArray() +// $data['metadata'] = $metadataQuery?->extractMetadata($data)?->asArray() ?? []; + //merge partial data if (!$data && is_string($file) && isset($this->partial[$file])) { return $this->partial[$file]; } elseif (!$data) { return $data; } else { + $data['metadata'] = $metadataQuery?->extractMetadata($data)?->jsonSerialize() ?? []; return self::cacheEntryFromData($data, $this->mimetypeLoader); } } @@ -239,11 +246,14 @@ public function getFolderContentsById($fileId) { ->whereParent($fileId) ->orderBy('name', 'ASC'); + $metadataQuery = $query->selectMetadata(); + $result = $query->execute(); $files = $result->fetchAll(); $result->closeCursor(); - return array_map(function (array $data) { + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery?->extractMetadata($data)?->jsonSerialize() ?? []; return self::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index 34d2177b84e9e..ccd8e85455e76 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -28,6 +28,8 @@ use OC\DB\QueryBuilder\QueryBuilder; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IMetadataQuery; use OCP\IDBConnection; use Psr\Log\LoggerInterface; @@ -35,9 +37,14 @@ * Query builder with commonly used helpers for filecache queries */ class CacheQueryBuilder extends QueryBuilder { - private $alias = null; - - public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) { + private ?string $alias = null; + + public function __construct( + IDBConnection $connection, + SystemConfig $systemConfig, + LoggerInterface $logger, + private IFilesMetadataManager $filesMetadataManager, + ) { parent::__construct($connection, $systemConfig, $logger); } @@ -126,4 +133,10 @@ public function whereParentInParameter(string $parameter) { return $this; } + + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 15c089a0f1147..b8a3aa0257c3d 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -25,9 +25,11 @@ */ namespace OC\Files\Cache; +use OC\DB\ConnectionAdapter; use OC\Files\Cache\Wrapper\CacheJail; use OC\Files\Search\QueryOptimizer\QueryOptimizer; use OC\Files\Search\SearchBinaryOperator; +use OC\FilesMetadata\Model\MetadataQuery; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICache; @@ -37,41 +39,24 @@ use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUser; use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Output\OutputInterface; class QuerySearchHelper { - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - /** @var IDBConnection */ - private $connection; - /** @var SystemConfig */ - private $systemConfig; - private LoggerInterface $logger; - /** @var SearchBuilder */ - private $searchBuilder; - /** @var QueryOptimizer */ - private $queryOptimizer; - private IGroupManager $groupManager; - public function __construct( - IMimeTypeLoader $mimetypeLoader, - IDBConnection $connection, - SystemConfig $systemConfig, - LoggerInterface $logger, - SearchBuilder $searchBuilder, - QueryOptimizer $queryOptimizer, - IGroupManager $groupManager, + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; - $this->connection = $connection; - $this->systemConfig = $systemConfig; - $this->logger = $logger; - $this->searchBuilder = $searchBuilder; - $this->queryOptimizer = $queryOptimizer; - $this->groupManager = $groupManager; } protected function getQueryBuilder() { @@ -144,6 +129,23 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): )); } + + protected function equipQueryForMetadata(CacheQueryBuilder $query, ISearchQuery $searchQuery): ?MetadataQuery { + // TODO: use $searchQuery to improve the query + + // init the thing + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'file', 'fileid'); + + // get metadata aside the files + $metadataQuery->retrieveMetadata(); + + // order by metadata photo_taken + $metadataQuery->leftJoinIndex('photo_taken'); + $query->orderBy($metadataQuery->getMetadataValueIntField(), 'desc'); + + return $metadataQuery; + } + /** * Perform a file system search in multiple caches * @@ -175,6 +177,7 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array $query = $builder->selectFileCache('file', false); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + if (in_array('systemtag', $requestedFields)) { $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } @@ -182,12 +185,16 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } + $metadataQuery = $this->equipQueryForMetadata($query, $searchQuery); $this->applySearchConstraints($query, $searchQuery, $caches); $result = $query->execute(); $files = $result->fetchAll(); - $rawEntries = array_map(function (array $data) { + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + // extract metadata from the result and convert it to array. + // TODO: using jsonSerialize() or another method that present each metadata as key => value ? + $data['metadata'] = $metadataQuery?->extractMetadata($data)?->asArray() ?? []; return Cache::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 7800074460b09..3fe9ba7d974c0 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -32,6 +32,7 @@ */ namespace OC\Files; +use OC\FilesMetadata\Model\FilesMetadata; use OCA\Files_Sharing\ISharedStorage; use OCP\Files\Cache\ICacheEntry; use OCP\Files\IHomeStorage; @@ -416,4 +417,8 @@ public function getUploadTime(): int { public function getParentId(): int { return $this->data['parent'] ?? -1; } + + public function getMetadata(): array { + return $this->data['metadata'] ?? []; + } } diff --git a/lib/private/Files/Node/LazyFolder.php b/lib/private/Files/Node/LazyFolder.php index f13cdc0c4f98f..ee0d1cfd3cfd9 100644 --- a/lib/private/Files/Node/LazyFolder.php +++ b/lib/private/Files/Node/LazyFolder.php @@ -574,4 +574,8 @@ public function getParentId(): int { } return $this->__call(__FUNCTION__, func_get_args()); } + + public function getMetadata(): array { + return $this->data['metadata'] ?? $this->__call(__FUNCTION__, func_get_args()); + } } diff --git a/lib/private/Files/Node/Node.php b/lib/private/Files/Node/Node.php index 9729f79aae3ca..c3bf50fa79587 100644 --- a/lib/private/Files/Node/Node.php +++ b/lib/private/Files/Node/Node.php @@ -43,7 +43,7 @@ use OCP\Lock\LockedException; use OCP\PreConditionNotMetException; -// FIXME: this class really should be abstract +// FIXME: this class really should be abstract (+1) class Node implements INode { /** * @var \OC\Files\View $view @@ -490,4 +490,8 @@ public function getUploadTime(): int { public function getParentId(): int { return $this->fileInfo->getParentId(); } + + public function getMetadata(): array { + return $this->fileInfo->getMetadata(); + } } diff --git a/lib/private/FilesMetadata/Event/MetadataEventBase.php b/lib/private/FilesMetadata/Event/MetadataEventBase.php new file mode 100644 index 0000000000000..758ce96d848ca --- /dev/null +++ b/lib/private/FilesMetadata/Event/MetadataEventBase.php @@ -0,0 +1,62 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Event; + +use OCP\Files\Node; +use OCP\EventDispatcher\Event; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataEventBase extends Event { + + public function __construct( + protected Node $node, + protected IFilesMetadata $metadata + ) { + parent::__construct(); + } + + /** + * returns id of the file + * + * @return Node + * @since 28.0.0 + */ + public function getNode(): Node { + return $this->node; + } + + /** + * returns Metadata model linked to file id, with already known metadata from the database. + * If the object is modified using its setters, metadata are updated in database at the end of the event. + * + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getMetadata(): IFilesMetadata { + return $this->metadata; + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php new file mode 100644 index 0000000000000..1a35f56999e22 --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -0,0 +1,163 @@ +metadataRequestService->getMetadataFromFileId($fileId); + } catch (FilesMetadataNotFoundException $e) { + if ($generate) { + return new FilesMetadata($fileId, true); + } + + throw $e; + } + } + + public function refreshMetadata( + Node $node, + int $process = self::PROCESS_LIVE, + bool $fromScratch = false, + ): IFilesMetadata { + $metadata = null; + if (!$fromScratch) { + try { + $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); + } catch (FilesMetadataNotFoundException $e) { + } + } + + if (null === $metadata) { + $metadata = new FilesMetadata($node->getId(), true); + } + + // is $process is LIVE, we enforce LIVE + if ((self::PROCESS_LIVE & $process) !== 0) { + $event = new MetadataLiveEvent($node, $metadata); + } else { + $event = new MetadataBackgroundEvent($node, $metadata); + } + + $this->eventDispatcher->dispatchTyped($event); + $this->saveMetadata($event->getMetadata()); + + // if requested, we add a new job for next cron to refresh metadata out of main thread + // if $process was set to LIVE+BACKGROUND, we run background process directly + if ($event instanceof MetadataLiveEvent && $event->isRunAsBackgroundJobRequested()) { + if ((self::PROCESS_BACKGROUND & $process) !== 0) { + return $this->refreshMetadata($node, self::PROCESS_BACKGROUND); + } + + $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]); + } + + return $metadata; + } + + /** + * @param IFilesMetadata $filesMetadata + * + * @return void + */ + public function saveMetadata(IFilesMetadata $filesMetadata): void { + if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) { + return; + } + + try { + // if update request changed no rows, means that new entry is needed, or sync_token not valid anymore + $updated = $this->metadataRequestService->updateMetadata($filesMetadata); + if ($updated === 0) { + $this->metadataRequestService->store($filesMetadata); + } + } catch (\OCP\DB\Exception $e) { + // if duplicate, only means a desync during update. cancel update process. + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + $this->logger->warning( + 'issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata] + ); + } + + return; + } + +// $this->removeDeprecatedMetadata($filesMetadata); + foreach ($filesMetadata->getIndexes() as $index) { + try { + $this->indexRequestService->updateIndex($filesMetadata, $index); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + } + + public function deleteMetadata(int $fileId): void { + try { + $this->metadataRequestService->dropMetadata($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + + try { + $this->indexRequestService->dropIndex($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + } + + public function getMetadataQuery( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField + ): IMetadataQuery { + return new MetadataQuery($qb, $fileTableAlias, $fileIdField); + } + + public static function loadListeners(IEventDispatcher $eventDispatcher): void { + $eventDispatcher->addServiceListener(NodeCreatedEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeDeletedEvent::class, MetadataDelete::class); + } +} diff --git a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php new file mode 100644 index 0000000000000..9824c9c1390be --- /dev/null +++ b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php @@ -0,0 +1,59 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Job; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\User\NoUserException; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; +use OCP\FilesMetadata\IFilesMetadataManager; + +class UpdateSingleMetadata extends QueuedJob { + public function __construct( + ITimeFactory $time, + private IRootFolder $rootFolder, + private FilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($time); + } + + protected function run($argument) { + [$userId, $fileId] = $argument; + + try { + // TODO: is there a way to get Node without $userId ? + $node = $this->rootFolder->getUserFolder($userId)->getById($fileId); + if (count($node) > 0) { + $file = array_shift($node); + $this->filesMetadataManager->refreshMetadata($file, IFilesMetadataManager::PROCESS_BACKGROUND); + } + } catch (NotPermittedException |NoUserException $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php new file mode 100644 index 0000000000000..ccc52d489cf33 --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -0,0 +1,60 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file deletion event and remove stored metadata related to the deleted file + */ +class MetadataDelete implements IEventListener { + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeDeletedEvent)) { + return; + } + + try { + $nodeId = (int)$event->getNode()->getId(); + if ($nodeId > 0) { + $this->filesMetadataManager->deleteMetadata($nodeId); + } + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php new file mode 100644 index 0000000000000..3e1d8ef9753cf --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -0,0 +1,61 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file creation/modification events and initiate a new event related to the created/edited file. + * The generated new event is broadcast in order to obtain file related metadata from other apps. + * metadata will be stored in database. + */ +class MetadataUpdate implements IEventListener { + + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeCreatedEvent) && !($event instanceof NodeWrittenEvent)) { + return; + } + + try { + $this->filesMetadataManager->refreshMetadata($event->getNode()); + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php new file mode 100644 index 0000000000000..1d63c5fde3b77 --- /dev/null +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -0,0 +1,374 @@ + */ + private array $metadata = []; + private int $lastUpdate = 0; + private string $syncToken = ''; + + public function __construct( + private int $fileId = 0, + private bool $updated = false + ) { + } + + public function getFileId(): int { + return $this->fileId; + } + + /** + * @param array $data + * + * @return IFilesMetadata + */ + public function import(array $data): IFilesMetadata { + foreach ($data as $k => $v) { + $valueWrapper = new MetadataValueWrapper(); + $this->metadata[$k] = $valueWrapper->import($v); + } + $this->updated = false; + + return $this; + } + + + /** + * import from database using the json field. + * + * if using aliases (ie. myalias_json), use $prefix='myalias_' + * + * @param array $data + * @param string $prefix + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function importFromDatabase(array $data, string $prefix = ''): IFilesMetadata { + try { + return $this->import( + json_decode($data[$prefix . 'json'] ?? '[]', + true, + 512, + JSON_THROW_ON_ERROR) + ); + } catch (JsonException $e) { + throw new FilesMetadataNotFoundException(); + } + } + + + public function updated(): bool { + return $this->updated; + } + + public function lastUpdateTimestamp(): int { + return $this->lastUpdate; + } + + public function getSyncToken(): string { + return $this->syncToken; + } + + public function hasKey(string $needle): bool { + return (in_array($needle, $this->getKeys())); + } + + public function getKeys(): array { + return array_keys($this->metadata); + } + + /** + * @return string[] + */ + public function getIndexes(): array { + $indexes = []; + foreach ($this->getKeys() as $key) { + if ($this->metadata[$key]->isIndexed()) { + $indexes[] = $key; + } + } + + return $indexes; + } + + /** + * @param string $key + * + * @return string + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function get(string $key): string { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueString(); + } + + /** + * @param string $key + * + * @return int + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getInt(string $key): int { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueInt(); + } + + /** + * @param string $key + * + * @return float + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getFloat(string $key): float { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueFloat(); + } + + /** + * @param string $key + * + * @return bool + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getBool(string $key): bool { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueBool(); + } + + /** + * @param string $key + * + * @return array + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getArray(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueArray(); + } + + /** + * @param string $key + * + * @return string[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getStringList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueStringList(); + } + + /** + * @param string $key + * + * @return int[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getIntList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueIntList(); + } + + public function getType(string $key): string { + return $this->metadata[$key]->getType(); + } + + public function set(string $key, string $value, bool $index = false): IFilesMetadata { + try { + if ($this->get($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value and index have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING); + $this->updated = true; + $this->metadata[$key] = $meta->setValueString($value)->setIndexed($index); + + return $this; + } + + public function setInt(string $key, int $value, bool $index = false): IFilesMetadata { + try { + if ($this->getInt($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_INT); + $this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setFloat(string $key, float $value, bool $index = false): IFilesMetadata { + try { + if ($this->getFloat($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_FLOAT); + $this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setBool(string $key, bool $value): IFilesMetadata { + try { + if ($this->getBool($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_BOOL); + $this->metadata[$key] = $meta->setValueBool($value); + $this->updated = true; + + return $this; + } + + public function setArray(string $key, array $value): IFilesMetadata { + try { + if ($this->getArray($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_ARRAY); + $this->metadata[$key] = $meta->setValueArray($value); + $this->updated = true; + + return $this; + } + + + public function setStringList(string $key, array $values, bool $index = false): IFilesMetadata { + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $meta->setValueStringList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setIntList(string $key, array $values, bool $index = false): IFilesMetadata { + $valueWrapper = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $valueWrapper->setValueIntList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function unset(string $key): IFilesMetadata { + unset($this->metadata[$key]); + $this->updated = true; + + return $this; + } + + public function removeStartsWith(string $keyPrefix): IFilesMetadata { + if ($keyPrefix === '') { + return $this; + } + + foreach ($this->getKeys() as $key) { + if (str_starts_with($key, $keyPrefix)) { + $this->unset($key); + } + } + + return $this; + } + + /** + * @param string $key + * + * @return MetadataValueWrapper + * @throws FilesMetadataNotFoundException + */ + public function getValueWrapper(string $key): MetadataValueWrapper { + if (!$this->hasKey($key)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]; + } + + + public function jsonSerialize(): array { + $data = []; + foreach ($this->metadata as $metaKey => $metaValueWrapper) { + $data[$metaKey] = $metaValueWrapper->jsonSerialize(); + } + + return $data; + } + + /** + * @return array + */ + public function asArray(): array { + $data = []; + foreach ($this->metadata as $metaKey => $metaValueWrapper) { + try { + $data[$metaKey] = $metaValueWrapper->getValueAny(); + } catch (FilesMetadataNotFoundException $e) { + // ignore exception + } + } + + return $data; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataQuery.php b/lib/private/FilesMetadata/Model/MetadataQuery.php new file mode 100644 index 0000000000000..b5914e7e7cec3 --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataQuery.php @@ -0,0 +1,108 @@ +queryBuilder->expr(); + $andX = $expr->andX($expr->eq($this->aliasIndex . '.file_id', $this->fileTableAlias . '.' . $this->fileIdField)); + + if ('' !== $metadataKey) { + $andX->add($expr->eq($this->getMetadataKeyField(), $this->queryBuilder->createNamedParameter($metadataKey))); + } + + $this->queryBuilder->leftJoin( + $this->fileTableAlias, + IndexRequestService::TABLE_METADATA_INDEX, + $this->aliasIndex, + $andX + ); + } + + + /** + * left join the metadata table to include a select of the stored json to the query + */ + public function retrieveMetadata(): void { + $this->queryBuilder->selectAlias($this->alias . '.json', 'meta_json'); + $this->queryBuilder->leftJoin( + $this->fileTableAlias, MetadataRequestService::TABLE_METADATA, $this->alias, + $this->queryBuilder->expr()->eq($this->fileTableAlias . '.' . $this->fileIdField, $this->alias . '.file_id') + ); + } + + public function enforceMetadataKey(string $metadataKey): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($metadataKey) + ) + ); + } + + public function enforceMetadataValue(string $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($value) + ) + ); + } + + public function enforceMetadataValueInt(int $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataValueIntField(), + $this->queryBuilder->createNamedParameter($value, IQueryBuilder::PARAM_INT) + ) + ); + } + + public function getMetadataKeyField(): string { + return $this->aliasIndex . '.meta_key'; + } + + public function getMetadataValueField(): string { + return $this->aliasIndex . '.meta_value'; + } + + public function getMetadataValueIntField(): string { + return $this->aliasIndex . '.meta_value_int'; + } + + public function extractMetadata(array $data): IFilesMetadata { + $fileId = (array_key_exists($this->fileIdField, $data)) ? $data[$this->fileIdField] : 0; + $metadata = new FilesMetadata($fileId); + $metadata->importFromDatabase($data, $this->alias . '_'); + + return $metadata; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php new file mode 100644 index 0000000000000..e4d6f1040e05d --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php @@ -0,0 +1,307 @@ +type = $type; + } + + public function setType(string $type): self { + $this->type = $type; + return $this; + } + + public function getType(): string { + return $this->type; + } + + public function isType(string $type): bool { + return (strtolower($type) === strtolower($this->type)); + } + + /** + * confirm stored value exists and is typed as requested + * @param string $type + * + * @return $this + * @throws FilesMetadataTypeException + */ + public function confirmType(string $type): self { + if (!$this->isType($type)) { + throw new FilesMetadataTypeException('type is \'' . $this->getType() . '\', expecting \'' . $type . '\''); + } + + return $this; + } + + /** + * @param string $value + * + * @return $this + */ + public function setValueString(string $value): self { + if ($this->isType(self::TYPE_STRING)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param int $value + * + * @return $this + */ + public function setValueInt(int $value): self { + if ($this->isType(self::TYPE_INT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param float $value + * + * @return $this + */ + public function setValueFloat(float $value): self { + if ($this->isType(self::TYPE_FLOAT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param bool $value + * + * @return $this + */ + public function setValueBool(bool $value): self { + if ($this->isType(self::TYPE_BOOL)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param array $value + * + * @return $this + */ + public function setValueArray(array $value): self { + if ($this->isType(self::TYPE_ARRAY)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param string[] $value + * + * @return $this + */ + public function setValueStringList(array $value): self { + if ($this->isType(self::TYPE_STRING_LIST)) { + // TODO confirm value is an array or string ? + $this->value = $value; + } + + return $this; + } + + /** + * @param int[] $value + * + * @return $this + */ + public function setValueIntList(array $value): self { + if ($this->isType(self::TYPE_INT_LIST)) { + // TODO confirm value is an array of int ? + $this->value = $value; + } + + return $this; + } + + + /** + * @return string + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueString(): string { + $this->confirmType(self::TYPE_STRING); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (string) $this->value; + } + + /** + * @return int + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueInt(): int { + $this->confirmType(self::TYPE_INT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (int) $this->value; + } + + /** + * @return float + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueFloat(): float { + $this->confirmType(self::TYPE_FLOAT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (float) $this->value; + } + + /** + * @return bool + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueBool(): bool { + $this->confirmType(self::TYPE_BOOL); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (bool) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueArray(): array { + $this->confirmType(self::TYPE_ARRAY); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return string[] + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueStringList(): array { + $this->confirmType(self::TYPE_STRING_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueIntList(): array { + $this->confirmType(self::TYPE_INT_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return string|int|float|bool|array|string[]|int[] + * @throws FilesMetadataNotFoundException + */ + public function getValueAny(): mixed { + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return $this->value; + } + + + public function setIndexed(bool $indexed): self { + $this->indexed = $indexed; + + return $this; + } + + public function isIndexed(): bool { + return $this->indexed; + } + + public function import(array $data): self { + $this->value = $data['value'] ?? null; + $this->setType($data['type'] ?? ''); + $this->setIndexed($data['indexed'] ?? false); + + return $this; + } + + public function jsonSerialize(): array { + return [ + 'value' => $this->value, + 'type' => $this->getType(), + 'indexed' => $this->isIndexed() + ]; + } +} diff --git a/lib/private/FilesMetadata/Provider/ExifMetadataProvider.php b/lib/private/FilesMetadata/Provider/ExifMetadataProvider.php new file mode 100644 index 0000000000000..6918d228c51f5 --- /dev/null +++ b/lib/private/FilesMetadata/Provider/ExifMetadataProvider.php @@ -0,0 +1,125 @@ + + * @copyright Copyright 2022 Louis Chmn + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\FilesMetadata\Provider; + +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\File; +use OCP\FilesMetadata\Event\MetadataLiveEvent; +use Psr\Log\LoggerInterface; + +// Extract EXIF, IFD0, and GPS data from a picture file. +// EXIF data reference: https://web.archive.org/web/20220428165430/exif.org/Exif2-2.PDF +class ExifMetadataProvider implements IEventListener { + public function __construct( + private LoggerInterface $logger + ) {} + + public function handle(Event $event): void { + if (!($event instanceof MetadataLiveEvent)) { + return; + } + + $node = $event->getNode(); + + if (!$node instanceof File) { + return; + } + + if (!preg_match('/image\/(png|jpeg|heif|webp|tiff)/', $node->getMimetype())) { + return; + } + + if (!extension_loaded('exif')) { + return; + } + + $fileDescriptor = $node->fopen('rb'); + if ($fileDescriptor === false) { + return; + } + + $rawExifData = null; + + try { + // HACK: The stream_set_chunk_size call is needed to make reading exif data reliable. + // This is to trigger this condition: https://github.com/php/php-src/blob/d64aa6f646a7b5e58359dc79479860164239580a/main/streams/streams.c#L710 + // But I don't understand yet why 1 as a special meaning. + $oldBufferSize = stream_set_chunk_size($fileDescriptor, 1); + $rawExifData = @exif_read_data($fileDescriptor, 'EXIF, GPS', true); + // We then revert the change after having read the exif data. + stream_set_chunk_size($fileDescriptor, $oldBufferSize); + } catch (\Exception $ex) { + $this->logger->info("Failed to extract metadata for " . $node->getId(), ['exception' => $ex]); + } + + if ($rawExifData && array_key_exists('EXIF', $rawExifData)) { + $event->getMetadata()->setArray('files-exif', $rawExifData['EXIF']); + } + + if ($rawExifData && array_key_exists('IFD0', $rawExifData)) { + $event->getMetadata()->setArray('files-ifd0', $rawExifData['IFD0']); + } + + if ( + $rawExifData && + array_key_exists('GPS', $rawExifData) && + array_key_exists('GPSLatitude', $rawExifData['GPS']) && array_key_exists('GPSLatitudeRef', $rawExifData['GPS']) && + array_key_exists('GPSLongitude', $rawExifData['GPS']) && array_key_exists('GPSLongitudeRef', $rawExifData['GPS']) + ) { + $event->getMetadata()->setArray('files-gps', [ + 'latitude' => $this->gpsDegreesToDecimal($rawExifData['GPS']['GPSLatitude'], $rawExifData['GPS']['GPSLatitudeRef']), + 'longitude' => $this->gpsDegreesToDecimal($rawExifData['GPS']['GPSLongitude'], $rawExifData['GPS']['GPSLongitudeRef']), + 'altitude' => ($rawExifData['GPS']['GPSAltitudeRef'] === "\u{0000}" ? 1 : -1) * $this->parseGPSData($rawExifData['GPS']['GPSAltitude']), + ]); + } + } + + /** + * @param array|string $coordinates + */ + private function gpsDegreesToDecimal($coordinates, ?string $hemisphere): float { + if (is_string($coordinates)) { + $coordinates = array_map("trim", explode(",", $coordinates)); + } + + if (count($coordinates) !== 3) { + throw new \Exception('Invalid coordinate format: ' . json_encode($coordinates)); + } + + [$degrees, $minutes, $seconds] = array_map(fn ($rawDegree) => $this->parseGPSData($rawDegree), $coordinates); + + $sign = ($hemisphere === 'W' || $hemisphere === 'S') ? -1 : 1; + return $sign * ($degrees + $minutes / 60 + $seconds / 3600); + } + + private function parseGPSData(string $rawData): float { + $parts = explode('/', $rawData); + + if ($parts[1] === '0') { + return 0; + } + + return floatval($parts[0]) / floatval($parts[1] ?? 1); + } +} diff --git a/lib/private/FilesMetadata/Provider/OriginalDateTimeMetadataProvider.php b/lib/private/FilesMetadata/Provider/OriginalDateTimeMetadataProvider.php new file mode 100644 index 0000000000000..373b6db29c9ce --- /dev/null +++ b/lib/private/FilesMetadata/Provider/OriginalDateTimeMetadataProvider.php @@ -0,0 +1,59 @@ + + * @copyright Copyright 2022 Louis Chmn + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\FilesMetadata\Provider; + +use DateTime; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\File; +use OCP\FilesMetadata\Event\MetadataLiveEvent; + +class OriginalDateTimeMetadataProvider implements IEventListener { + public function __construct() {} + + public function handle(Event $event): void { + if (!($event instanceof MetadataLiveEvent)) { + return; + } + + $node = $event->getNode(); + + if (!$node instanceof File) { + return; + } + + $metadata = $event->getMetadata(); + + if (!$metadata->hasKey('files-exif')) { + return; + } + + if (!array_key_exists('DateTimeOriginal', $metadata->getArray('files-exif'))) { + return; + } + + $rawDateTimeOriginal = $metadata->getArray('files-exif')['DateTimeOriginal']; + $dateTimeOriginal = DateTime::createFromFormat("Y:m:d G:i:s", $rawDateTimeOriginal); + $metadata->setInt('files-original_date_time', $dateTimeOriginal->getTimestamp(), true); + } +} diff --git a/lib/private/FilesMetadata/Provider/SizeMetadataProvider.php b/lib/private/FilesMetadata/Provider/SizeMetadataProvider.php new file mode 100644 index 0000000000000..c2bc42aa86ae8 --- /dev/null +++ b/lib/private/FilesMetadata/Provider/SizeMetadataProvider.php @@ -0,0 +1,62 @@ + + * @copyright Copyright 2022 Louis Chmn + * @license AGPL-3.0-or-later + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\FilesMetadata\Provider; + +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\File; +use OCP\FilesMetadata\Event\MetadataLiveEvent; +use Psr\Log\LoggerInterface; + +class SizeMetadataProvider implements IEventListener { + public function __construct( + private LoggerInterface $logger + ) {} + + public function handle(Event $event): void { + if (!($event instanceof MetadataLiveEvent)) { + return; + } + + $node = $event->getNode(); + + if (!$node instanceof File) { + return; + } + + if (!preg_match('/image\/(png|jpeg|heif|webp|tiff)/', $node->getMimetype())) { + return; + } + + $size = getimagesizefromstring($node->getContent()); + + if ($size === false) { + return; + } + + $event->getMetadata()->setArray('files-size', [ + 'width' => $size[0], + 'height' => $size[1], + ]); + } +} diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php new file mode 100644 index 0000000000000..6804625de3235 --- /dev/null +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -0,0 +1,131 @@ +getFileId(); + + /** + * might look harsh, but a lot simpler than comparing current indexed data, as we can expect + * conflict with a change of types. + * We assume that each time one random metadata were modified we can drop all index for this + * key and recreate them + */ + $this->dropIndex($fileId, $key); + + try { + match ($filesMetadata->getType($key)) { + MetadataValueWrapper::TYPE_STRING + => $this->insertIndexString($fileId, $key, $filesMetadata->get($key)), + MetadataValueWrapper::TYPE_INT + => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)), + MetadataValueWrapper::TYPE_STRING_LIST + => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)), + MetadataValueWrapper::TYPE_INT_LIST + => $this->insertIndexIntList($fileId, $key, $filesMetadata->getIntList($key)) + }; + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + $this->logger->warning('...'); + } + } + + + private function insertIndexString(int $fileId, string $key, string $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value', $qb->createNamedParameter($value)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + public function insertIndexInt(int $fileId, string $key, int $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + /** + * @param int $fileId + * @param string $key + * @param string[] $values + * + * @return void + */ + public function insertIndexStringList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexString($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * @param int[] $values + * + * @return void + */ + public function insertIndexIntList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexInt($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * + * @return void + * @throws Exception + */ + public function dropIndex(int $fileId, string $key = ''): void { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + if ($key !== '') { + $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); + } + + $qb->executeStatement(); + } +} diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php new file mode 100644 index 0000000000000..0c82d80b760a8 --- /dev/null +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -0,0 +1,113 @@ +dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) + ->setValue('last_update', $qb->createFunction('NOW()')); + $qb->executeStatement(); + } + + /** + * @param int $fileId + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function getMetadataFromFileId(int $fileId): IFilesMetadata { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('json')->from(self::TABLE_METADATA); + $qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + } catch (Exception $e) { + $this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]); + throw new FilesMetadataNotFoundException(); + } + + if ($data === false) { + throw new FilesMetadataNotFoundException(); + } + + $metadata = new FilesMetadata($fileId); + $metadata->importFromDatabase($data); + + return $metadata; + } + + + /** + * @param int $fileId + * + * @return void + * @throws Exception + */ + public function dropMetadata(int $fileId): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } + + private function removeDeprecatedMetadata(IFilesMetadata $filesMetadata): void { + // TODO delete aussi les index generate a partir d'une string[] + + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->notIn('file_id', $filesMetadata->getIndexes(), IQueryBuilder::PARAM_STR_ARRAY)); + $qb->executeStatement(); + } + + + /** + * @param IFilesMetadata $filesMetadata + * + * @return bool + * @throws Exception + */ + public function updateMetadata(IFilesMetadata $filesMetadata): int { + // TODO check sync_token on update + $qb = $this->dbConnection->getQueryBuilder(); + $qb->update(self::TABLE_METADATA) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter('abc')) + ->set('last_update', $qb->createFunction('NOW()')) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + + return $qb->executeStatement(); + } +} diff --git a/lib/private/Metadata/MetadataManager.php b/lib/private/Metadata/MetadataManager.php index 6d96ff1ab6806..0398fe607fcbd 100644 --- a/lib/private/Metadata/MetadataManager.php +++ b/lib/private/Metadata/MetadataManager.php @@ -19,7 +19,6 @@ namespace OC\Metadata; -use OC\Metadata\Provider\ExifProvider; use OCP\Files\File; class MetadataManager implements IMetadataManager { @@ -34,9 +33,6 @@ public function __construct( $this->providers = []; $this->providerClasses = []; $this->fileMetadataMapper = $fileMetadataMapper; - - // TODO move to another place, where? - $this->registerProvider(ExifProvider::class); } /** @@ -62,16 +58,16 @@ public function generateMetadata(File $file, bool $checkExisting = false): void } } - foreach ($this->providers as $supportedMimetype => $provider) { - if (preg_match($supportedMimetype, $file->getMimeType())) { - if (count(array_diff($provider::groupsProvided(), $existingMetadataGroups)) > 0) { - $metaDataGroup = $provider->execute($file); - foreach ($metaDataGroup as $group => $metadata) { - $this->fileMetadataMapper->insertOrUpdate($metadata); - } - } - } - } + // foreach ($this->providers as $supportedMimetype => $provider) { + // if (preg_match($supportedMimetype, $file->getMimeType())) { + // if (count(array_diff($provider::groupsProvided(), $existingMetadataGroups)) > 0) { + // $metaDataGroup = $provider->execute($file); + // foreach ($metaDataGroup as $group => $metadata) { + // $this->fileMetadataMapper->insertOrUpdate($metadata); + // } + // } + // } + // } } public function clearMetadata(int $fileId): void { diff --git a/lib/private/Metadata/Provider/ExifProvider.php b/lib/private/Metadata/Provider/ExifProvider.php deleted file mode 100644 index b1598abbbc8b7..0000000000000 --- a/lib/private/Metadata/Provider/ExifProvider.php +++ /dev/null @@ -1,142 +0,0 @@ - - * @copyright Copyright 2022 Louis Chmn - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata\Provider; - -use OC\Metadata\FileMetadata; -use OC\Metadata\IMetadataProvider; -use OCP\Files\File; -use Psr\Log\LoggerInterface; - -class ExifProvider implements IMetadataProvider { - private LoggerInterface $logger; - - public function __construct( - LoggerInterface $logger - ) { - $this->logger = $logger; - } - - public static function groupsProvided(): array { - return ['size', 'gps']; - } - - public static function isAvailable(): bool { - return extension_loaded('exif'); - } - - /** @return array{'gps'?: FileMetadata, 'size'?: FileMetadata} */ - public function execute(File $file): array { - $exifData = []; - $fileDescriptor = $file->fopen('rb'); - - if ($fileDescriptor === false) { - return []; - } - - $data = null; - try { - // Needed to make reading exif data reliable. - // This is to trigger this condition: https://github.com/php/php-src/blob/d64aa6f646a7b5e58359dc79479860164239580a/main/streams/streams.c#L710 - // But I don't understand why 1 as a special meaning. - // Revert right after reading the exif data. - $oldBufferSize = stream_set_chunk_size($fileDescriptor, 1); - $data = @exif_read_data($fileDescriptor, 'ANY_TAG', true); - stream_set_chunk_size($fileDescriptor, $oldBufferSize); - } catch (\Exception $ex) { - $this->logger->info("Couldn't extract metadata for ".$file->getId(), ['exception' => $ex]); - } - - $size = new FileMetadata(); - $size->setGroupName('size'); - $size->setId($file->getId()); - $size->setArrayAsValue([]); - - if (!$data) { - $sizeResult = getimagesizefromstring($file->getContent()); - if ($sizeResult !== false) { - $size->setArrayAsValue([ - 'width' => $sizeResult[0], - 'height' => $sizeResult[1], - ]); - - $exifData['size'] = $size; - } - } elseif (array_key_exists('COMPUTED', $data)) { - if (array_key_exists('Width', $data['COMPUTED']) && array_key_exists('Height', $data['COMPUTED'])) { - $size->setArrayAsValue([ - 'width' => $data['COMPUTED']['Width'], - 'height' => $data['COMPUTED']['Height'], - ]); - - $exifData['size'] = $size; - } - } - - if ($data && array_key_exists('GPS', $data) - && array_key_exists('GPSLatitude', $data['GPS']) && array_key_exists('GPSLatitudeRef', $data['GPS']) - && array_key_exists('GPSLongitude', $data['GPS']) && array_key_exists('GPSLongitudeRef', $data['GPS']) - ) { - $gps = new FileMetadata(); - $gps->setGroupName('gps'); - $gps->setId($file->getId()); - $gps->setArrayAsValue([ - 'latitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLatitude'], $data['GPS']['GPSLatitudeRef']), - 'longitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLongitude'], $data['GPS']['GPSLongitudeRef']), - ]); - - $exifData['gps'] = $gps; - } - - return $exifData; - } - - public static function getMimetypesSupported(): string { - return '/image\/(png|jpeg|heif|webp|tiff)/'; - } - - /** - * @param array|string $coordinates - */ - private static function gpsDegreesToDecimal($coordinates, ?string $hemisphere): float { - if (is_string($coordinates)) { - $coordinates = array_map("trim", explode(",", $coordinates)); - } - - if (count($coordinates) !== 3) { - throw new \Exception('Invalid coordinate format: ' . json_encode($coordinates)); - } - - [$degrees, $minutes, $seconds] = array_map(function (string $rawDegree) { - $parts = explode('/', $rawDegree); - - if ($parts[1] === '0') { - return 0; - } - - return floatval($parts[0]) / floatval($parts[1] ?? 1); - }, $coordinates); - - $sign = ($hemisphere === 'W' || $hemisphere === 'S') ? -1 : 1; - return $sign * ($degrees + $minutes / 60 + $seconds / 3600); - } -} diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 4a1270fa4a60b..5a68b01523475 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -37,6 +37,7 @@ use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\ISimpleFile; use OCP\Files\SimpleFS\ISimpleFolder; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -44,6 +45,7 @@ use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; +use kornrunner\Blurhash\Blurhash; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -65,7 +67,8 @@ public function __construct( IPreview $previewManager, IAppData $appData, GeneratorHelper $helper, - IEventDispatcher $eventDispatcher + IEventDispatcher $eventDispatcher, + private IFilesMetadataManager $filesMetadataManager, ) { $this->config = $config; $this->previewManager = $previewManager; @@ -191,6 +194,9 @@ public function generatePreviews(File $file, array $specifications, $mimeType = } $preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion); + if ($width === 256) { + $this->saveBlurhash($file->getId(), $width, $height, $preview->getContent(), $crop); + } // New file, augment our array $previewFiles[] = $preview; } @@ -623,4 +629,27 @@ private function getExtension($mimeType) { throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); } } + + private function saveBlurhash(int $fileId, int $width, int $height, string $content, bool $crop): void { + $image = imagecreatefromstring($content); + + $pixels = []; + for ($y = 0; $y < $height; ++$y) { + $row = []; + for ($x = 0; $x < $width; ++$x) { + $index = imagecolorat($image, $x, $y); + $colors = imagecolorsforindex($image, $index); + + $row[] = [$colors['red'], $colors['green'], $colors['blue']]; + } + $pixels[] = $row; + } + + $metadata = $this->filesMetadataManager->getMetadata($fileId); + if ($crop) { + $metadata->set('files-blurhash-crop', Blurhash::encode($pixels, 4, 3), true); + } else { + $metadata->set('files-blurhash', Blurhash::encode($pixels, 4, 3), true); + } + } } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 3af6848686ee7..de55b22888856 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -41,6 +41,7 @@ use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IBinaryFinder; use OCP\IConfig; use OCP\IPreview; @@ -75,16 +76,17 @@ class PreviewManager implements IPreview { private IMagickSupport $imagickSupport; public function __construct( - IConfig $config, - IRootFolder $rootFolder, - IAppData $appData, - IEventDispatcher $eventDispatcher, - GeneratorHelper $helper, - ?string $userId, - Coordinator $bootstrapCoordinator, - IServerContainer $container, - IBinaryFinder $binaryFinder, - IMagickSupport $imagickSupport + IConfig $config, + IRootFolder $rootFolder, + IAppData $appData, + IEventDispatcher $eventDispatcher, + GeneratorHelper $helper, + ?string $userId, + Coordinator $bootstrapCoordinator, + IServerContainer $container, + IBinaryFinder $binaryFinder, + IMagickSupport $imagickSupport, + private IFilesMetadataManager $filesMetadataManager ) { $this->config = $config; $this->rootFolder = $rootFolder; @@ -157,7 +159,8 @@ private function getGenerator(): Generator { $this->rootFolder, $this->config ), - $this->eventDispatcher + $this->eventDispatcher, + $this->filesMetadataManager, ); } return $this->generator; diff --git a/lib/private/Server.php b/lib/private/Server.php index 949a7ccfd3f4d..0cf5f94fb36cb 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -102,6 +102,7 @@ use OC\Files\Template\TemplateManager; use OC\Files\Type\Loader; use OC\Files\View; +use OC\FilesMetadata\FilesMetadataManager; use OC\FullTextSearch\FullTextSearchManager; use OC\Http\Client\ClientService; use OC\Http\Client\NegativeDnsCache; @@ -194,6 +195,7 @@ use OCP\Files\Mount\IMountManager; use OCP\Files\Storage\IStorageFactory; use OCP\Files\Template\ITemplateManager; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\GlobalScale\IConfig; use OCP\Group\ISubAdmin; @@ -342,7 +344,8 @@ public function __construct($webRoot, \OC\Config $config) { $c->get(Coordinator::class), $c->get(IServerContainer::class), $c->get(IBinaryFinder::class), - $c->get(IMagickSupport::class) + $c->get(IMagickSupport::class), + $c->get(IFilesMetadataManager::class), ); }); /** @deprecated 19.0.0 */ @@ -1397,6 +1400,7 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(\OCP\Dashboard\IManager::class, \OC\Dashboard\Manager::class); $this->registerAlias(IFullTextSearchManager::class, FullTextSearchManager::class); + $this->registerAlias(IFilesMetadataManager::class, FilesMetadataManager::class); $this->registerAlias(ISubAdmin::class, SubAdmin::class); @@ -1470,6 +1474,8 @@ private function connectDispatcher(): void { $eventDispatcher->addServiceListener(PostLoginEvent::class, UserLoggedInListener::class); $eventDispatcher->addServiceListener(UserChangedEvent::class, UserChangedListener::class); $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); + + FilesMetadataManager::loadListeners($eventDispatcher); } /** diff --git a/lib/public/Files/FileInfo.php b/lib/public/Files/FileInfo.php index da35f7f90283b..f16d2d7f01951 100644 --- a/lib/public/Files/FileInfo.php +++ b/lib/public/Files/FileInfo.php @@ -28,6 +28,7 @@ */ namespace OCP\Files; +use OC\FilesMetadata\Model\MetadataValueWrapper; use OCP\Files\Storage\IStorage; /** @@ -308,4 +309,11 @@ public function getUploadTime(): int; * @since 28.0.0 */ public function getParentId(): int; + + /** + * Get the metadata, if available + * + * @since 28.0.0 + */ + public function getMetadata(): array; } diff --git a/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php new file mode 100644 index 0000000000000..320d98c8edfe1 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php @@ -0,0 +1,40 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataBackgroundEvent extends MetadataEventBase { + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } +} diff --git a/lib/public/FilesMetadata/Event/MetadataLiveEvent.php b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php new file mode 100644 index 0000000000000..c9c2306368566 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php @@ -0,0 +1,64 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataLiveEvent extends MetadataEventBase { + private bool $runAsBackgroundJob = false; + + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } + + /** + * If an app prefer to update metadata on a background job, instead of + * live process, just call this method. + * A new event will be generated on next cron tick. + * + * @return void + * @since 28.0.0 + */ + public function requestBackgroundJob(): void { + $this->runAsBackgroundJob = true; + } + + /** + * return true if any app that catch this event requested a re-run as background job + * + * @return bool + * @since 28.0.0 + */ + public function isRunAsBackgroundJobRequested(): bool { + return $this->runAsBackgroundJob; + } +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php new file mode 100644 index 0000000000000..93685e22becf7 --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php @@ -0,0 +1,10 @@ +