transform favorite, tag and systemtag search queries to fileid lists

not as performant but compatible with sharding

Signed-off-by: Robin Appelman <[email protected]>

lib/private/Files/Cache/QuerySearchHelper.php

@@ -24,11 +24,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  */
+
 namespace OC\Files\Cache;
 
 use OC\Files\Cache\Wrapper\CacheJail;
+use OC\Files\Search\QueryOptimizer\DavTagToFileIds;
+use OC\Files\Search\QueryOptimizer\FavoriteToTag;
 use OC\Files\Search\QueryOptimizer\QueryOptimizer;
+use OC\Files\Search\QueryOptimizer\SystemTagToFileIds;
 use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
 use OC\SystemConfig;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\ICache;
@@ -47,13 +53,13 @@
 
 class QuerySearchHelper {
 	public function __construct(
-		private IMimeTypeLoader $mimetypeLoader,
-		private IDBConnection $connection,
-		private SystemConfig $systemConfig,
-		private LoggerInterface $logger,
-		private SearchBuilder $searchBuilder,
-		private QueryOptimizer $queryOptimizer,
-		private IGroupManager $groupManager,
+		private IMimeTypeLoader       $mimetypeLoader,
+		private IDBConnection         $connection,
+		private SystemConfig          $systemConfig,
+		private LoggerInterface       $logger,
+		private SearchBuilder         $searchBuilder,
+		private QueryOptimizer        $queryOptimizer,
+		private IGroupManager         $groupManager,
 		private IFilesMetadataManager $filesMetadataManager,
 		private CacheDatabase $cacheDatabase,
 	) {
@@ -76,9 +82,9 @@ protected function getQueryBuilder() {
 	 */
 	protected function applySearchConstraints(
 		CacheQueryBuilder $query,
-		ISearchQuery $searchQuery,
-		array $caches,
-		?IMetadataQuery $metadataQuery = null
+		ISearchQuery      $searchQuery,
+		array             $caches,
+		?IMetadataQuery   $metadataQuery = null
 	): void {
 		$storageFilters = array_values(array_map(function (ICache $cache) {
 			return $cache->getQueryFilterForStorage();
@@ -132,18 +138,6 @@ protected function equipQueryForSystemTags(CacheQueryBuilder $query, IUser $user
 		$query->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on);
 	}
 
-	protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): void {
-		$query
-			->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid'))
-			->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
-				$query->expr()->eq('tagmap.type', 'tag.type'),
-				$query->expr()->eq('tagmap.categoryid', 'tag.id'),
-				$query->expr()->eq('tag.type', $query->createNamedParameter('files')),
-				$query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID()))
-			));
-	}
-
-
 	protected function equipQueryForShares(CacheQueryBuilder $query): void {
 		$query->join('file', 'share', 's', $query->expr()->eq('file.fileid', 's.file_source'));
 	}
@@ -174,20 +168,32 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
 		// while the resulting rows don't have a way to tell what storage they came from (multiple storages/caches can share storage_id)
 		// we can just ask every cache if the row belongs to them and give them the cache to do any post processing on the result.
 
-		$storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches);
-		$cachesByStorage = array_combine($storageIds, $caches);
+		$operation = $searchQuery->getSearchOperation();
 
 		$requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation());
 
+		if (in_array('favorite', $requestedFields)) {
+			$favoriteHandler = new FavoriteToTag();
+			$favoriteHandler->processOperator($operation);
+		}
+
+		if (in_array('systemtag', $requestedFields)) {
+			$systemTagHandler = new SystemTagToFileIds($this->connection, $this->requireUser($searchQuery), $this->groupManager);
+			$systemTagHandler->processOperator($operation);
+		}
+		if (in_array('tagname', $requestedFields)) {
+			$davTagHandler = new DavTagToFileIds($this->connection, $this->requireUser($searchQuery));
+			$davTagHandler->processOperator($operation);
+		}
+
+		$searchQuery->setSearchOperation($operation);
+
+		$storageIds = array_map(fn (ICache $cache) => $cache->getNumericStorageId(), $caches);
+		$cachesByStorage = array_combine($storageIds, $caches);
+
 		$rawEntries = $this->cacheDatabase->queryStorages($storageIds, function(CacheQueryBuilder $builder, $storages) use ($requestedFields, $searchQuery, $cachesByStorage) {
 			$cachesForShard = array_map(fn (int $storage) => $cachesByStorage[$storage], $storages);
 			$query = $builder->selectFileCache('file', false);
-			if (in_array('systemtag', $requestedFields)) {
-				$this->equipQueryForSystemTags($query, $this->requireUser($searchQuery));
-			}
-			if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) {
-				$this->equipQueryForDavTags($query, $this->requireUser($searchQuery));
-			}
 			if (in_array('owner', $requestedFields) || in_array('share_with', $requestedFields) || in_array('share_type', $requestedFields)) {
 				$this->equipQueryForShares($query);
 			}

lib/private/Files/Cache/SearchBuilder.php

@@ -75,9 +75,6 @@ class SearchBuilder {
 		'name' => 'string',
 		'path' => 'string',
 		'size' => 'integer',
-		'tagname' => 'string',
-		'systemtag' => 'string',
-		'favorite' => 'boolean',
 		'fileid' => 'integer',
 		'storage' => 'integer',
 		'share_with' => 'string',
@@ -99,8 +96,6 @@ class SearchBuilder {
 		'boolean' => IQueryBuilder::PARAM_INT_ARRAY,
 	];
 
-	public const TAG_FAVORITE = '_$!<Favorite>!$_';
-
 	/** @var IMimeTypeLoader */
 	private $mimetypeLoader;
 
@@ -243,16 +238,8 @@ private function getOperatorFieldAndValueInner(string $field, mixed $value, stri
 					$type = ISearchComparison::COMPARE_EQUAL;
 				}
 			}
-		} elseif ($field === 'favorite') {
-			$field = 'tag.category';
-			$value = self::TAG_FAVORITE;
-			$paramType = 'string';
 		} elseif ($field === 'name') {
 			$field = 'file.name';
-		} elseif ($field === 'tagname') {
-			$field = 'tag.category';
-		} elseif ($field === 'systemtag') {
-			$field = 'systemtag.name';
 		} elseif ($field === 'fileid') {
 			$field = 'file.fileid';
 		} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) {
@@ -271,9 +258,6 @@ private function validateComparison(ISearchComparison $operator) {
 			'name' => ['eq', 'like', 'clike', 'in'],
 			'path' => ['eq', 'like', 'clike', 'in'],
 			'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
-			'tagname' => ['eq', 'like'],
-			'systemtag' => ['eq', 'like'],
-			'favorite' => ['eq'],
 			'fileid' => ['eq', 'in'],
 			'storage' => ['eq', 'in'],
 			'share_with' => ['eq'],

lib/private/Files/Search/QueryOptimizer/DavTagToFileIds.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class DavTagToFileIds extends ReplacingOptimizerStep {
+	public function __construct(
+		private IDBConnection $connection,
+		private IUser $user,
+	) {
+	}
+
+	public function processOperator(ISearchOperator &$operator): bool {
+		if ($operator instanceof ISearchComparison && $operator->getField() === 'tagname') {
+			$operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForDavTag($operator));
+			return true;
+		} else {
+			return parent::processOperator($operator);
+		}
+	}
+
+
+	private function getFileIdsForDavTag(ISearchComparison $comparison): array {
+		$query = $this->connection->getQueryBuilder();
+
+		$query->select('tagmap.objid')
+			->from('vcategory_to_object', 'tagmap')
+			->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
+				$query->expr()->eq('tagmap.type', 'tag.type'),
+				$query->expr()->eq('tagmap.categoryid', 'tag.id')
+			))
+			->where($query->expr()->eq('tag.type', $query->createNamedParameter('files')))
+			->andWhere($query->expr()->eq('tag.uid', $query->createNamedParameter($this->user->getUID())));
+		if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) {
+			$query->andWhere($query->expr()->eq('tag.category', $query->createNamedParameter($comparison->getValue())));
+		} elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) {
+			$query->andWhere($query->expr()->like('tag.category', $query->createNamedParameter($comparison->getValue())));
+		} else {
+			throw new \InvalidArgumentException('Unsupported comparison for field  ' . $comparison->getField() . ': ' . $comparison->getType());
+		}
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+	}
+}

lib/private/Files/Search/QueryOptimizer/FavoriteToTag.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class FavoriteToTag extends ReplacingOptimizerStep {
+	public const TAG_FAVORITE = '_$!<Favorite>!$_';
+
+	public function processOperator(ISearchOperator &$operator): bool {
+		if ($operator instanceof ISearchComparison && $operator->getField() === 'favorite' && $operator->getType() === ISearchComparison::COMPARE_EQUAL) {
+			$operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', self::TAG_FAVORITE);
+			return true;
+		} else {
+			return parent::processOperator($operator);
+		}
+	}
+}

lib/private/Files/Search/QueryOptimizer/SystemTagToFileIds.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2024 Robin Appelman <[email protected]>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchComparison;
+use OCP\Files\Search\ISearchComparison;
+use OCP\Files\Search\ISearchOperator;
+use OCP\IDBConnection;
+use OCP\IGroupManager;
+use OCP\IUser;
+
+class SystemTagToFileIds extends ReplacingOptimizerStep {
+	public function __construct(
+		private IDBConnection $connection,
+		private IUser $user,
+		private IGroupManager $groupManager,
+	) {
+	}
+
+	public function processOperator(ISearchOperator &$operator): bool {
+		if ($operator instanceof ISearchComparison && $operator->getField() === 'systemtag') {
+			$operator = new SearchComparison(ISearchComparison::COMPARE_IN, 'fileid', $this->getFileIdsForSystemTag($operator));
+			return true;
+		} else {
+			return parent::processOperator($operator);
+		}
+	}
+
+
+	private function getFileIdsForSystemTag(ISearchComparison $comparison): array {
+		$query = $this->connection->getQueryBuilder();
+
+		$on = $query->expr()->andX($query->expr()->eq('systemtag.id', 'systemtagmap.systemtagid'));
+		if (!$this->groupManager->isAdmin($this->user->getUID())) {
+			$on->add($query->expr()->eq('systemtag.visibility', $query->createNamedParameter(true)));
+		}
+
+		$query->select('systemtagmap.objectid')
+			->from('systemtag_object_mapping', 'systemtagmap')
+			->leftJoin('systemtagmap', 'systemtag', 'systemtag', $on)
+			->where($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')))
+			->andWhere($query->expr()->eq('systemtagmap.objecttype', $query->createNamedParameter('files')));
+		if ($comparison->getType() === ISearchComparison::COMPARE_EQUAL) {
+			$query->andWhere($query->expr()->eq('systemtag.name', $query->createNamedParameter($comparison->getValue())));
+		} elseif ($comparison->getType() === ISearchComparison::COMPARE_LIKE) {
+			$query->andWhere($query->expr()->like('systemtag.name', $query->createNamedParameter($comparison->getValue())));
+		} else {
+			throw new \InvalidArgumentException('Unsupported comparison for field  ' . $comparison->getField() . ': ' . $comparison->getType());
+		}
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+	}
+}

lib/private/Files/Search/SearchQuery.php

@@ -73,6 +73,10 @@ public function getSearchOperation() {
 		return $this->searchOperation;
 	}
 
+	public function setSearchOperation(ISearchOperator $operator): void {
+		$this->searchOperation = $operator;
+	}
+
 	/**
 	 * @return int
 	 */

lib/public/Files/Search/ISearchQuery.php

@@ -34,6 +34,13 @@ interface ISearchQuery {
 	 */
 	public function getSearchOperation();
 
+	/**
+	 * @param ISearchOperator $operator
+	 * @return void
+	 * @since 29.0.0
+	 */
+	public function setSearchOperation(ISearchOperator $operator): void;
+
 	/**
 	 * Get the maximum number of results to return
 	 *