optimize query pattern used by storage filter

Signed-off-by: Robin Appelman <[email protected]>

lib/composer/composer/LICENSE

@@ -1,4 +1,3 @@
-
 Copyright (c) Nils Adermann, Jordi Boggiano
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
-

lib/composer/composer/autoload_classmap.php

@@ -1406,9 +1406,14 @@
     'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
     'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
     'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => $baseDir . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
+    'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
+    'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
+    'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
+    'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
     'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
     'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
     'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
+    'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
     'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
     'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php',
     'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php',

lib/composer/composer/autoload_static.php

@@ -1439,9 +1439,14 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
         'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
         'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
+        'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
+        'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
+        'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
+        'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
         'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
         'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
         'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
+        'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
         'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
         'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php',
         'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php',

lib/private/Files/Cache/SearchBuilder.php

@@ -48,6 +48,7 @@ class SearchBuilder {
 		ISearchComparison::COMPARE_LESS_THAN => 'lt',
 		ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte',
 		ISearchComparison::COMPARE_DEFINED => 'isNotNull',
+		ISearchComparison::COMPARE_IN => 'in',
 	];
 
 	protected static $searchOperatorNegativeMap = [
@@ -59,6 +60,34 @@ class SearchBuilder {
 		ISearchComparison::COMPARE_LESS_THAN => 'gte',
 		ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt',
 		ISearchComparison::COMPARE_DEFINED => 'isNull',
+		ISearchComparison::COMPARE_IN => 'notIn',
+	];
+
+	protected static $fieldTypes = [
+		'mimetype' => 'string',
+		'mtime' => 'integer',
+		'name' => 'string',
+		'path' => 'string',
+		'size' => 'integer',
+		'tagname' => 'string',
+		'systemtag' => 'string',
+		'favorite' => 'boolean',
+		'fileid' => 'integer',
+		'storage' => 'integer',
+		'share_with' => 'string',
+		'share_type' => 'integer',
+		'owner' => 'string',
+	];
+
+	protected static $paramTypeMap = [
+		'string' => IQueryBuilder::PARAM_STR,
+		'integer' => IQueryBuilder::PARAM_INT,
+		'boolean' => IQueryBuilder::PARAM_BOOL,
+	];
+	protected static $paramArrayTypeMap = [
+		'string' => IQueryBuilder::PARAM_STR_ARRAY,
+		'integer' => IQueryBuilder::PARAM_INT_ARRAY,
+		'boolean' => IQueryBuilder::PARAM_INT_ARRAY,
 	];
 
 	public const TAG_FAVORITE = '_$!<Favorite>!$_';
@@ -142,31 +171,56 @@ private function searchComparisonToDBExpr(
 		?IMetadataQuery $metadataQuery = null
 	) {
 		if ($comparison->getExtra()) {
-			[$field, $value, $type] = $this->getExtraOperatorField($comparison, $metadataQuery);
+			[$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery);
 		} else {
-			[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
+			[$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison);
 		}
 
 		if (isset($operatorMap[$type])) {
 			$queryOperator = $operatorMap[$type];
-			return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
+			return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType));
 		} else {
 			throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType());
 		}
 	}
 
-	private function getOperatorFieldAndValue(ISearchComparison $operator) {
+	/**
+	 * @param ISearchComparison $operator
+	 * @return list{string, string|integer|\DateTime|(\DateTime|int|string)[], string, string}
+	 */
+	private function getOperatorFieldAndValue(ISearchComparison $operator): array {
 		$this->validateComparison($operator);
-
 		$field = $operator->getField();
 		$value = $operator->getValue();
 		$type = $operator->getType();
+		$pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true);
+		return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash);
+	}
 
+	/**
+	 * @param string $field
+	 * @param string|integer|\DateTime|(\DateTime|int|string)[] $value
+	 * @param string $type
+	 * @return list{string, string|integer|\DateTime|(\DateTime|int|string)[], string, string}
+	 */
+	private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array {
+		$paramType = self::$fieldTypes[$field];
+		if ($type === ISearchComparison::COMPARE_IN) {
+			$resultField = $field;
+			$values = [];
+			foreach ($value as $arrayValue) {
+				/** @var string|integer|\DateTime $arrayValue */
+				[$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash);
+				$resultField = $arrayField;
+				$values[] = $arrayValue;
+			}
+			return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType];
+		}
 		if ($field === 'mimetype') {
 			$value = (string)$value;
-			if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
+			if ($type === ISearchComparison::COMPARE_EQUAL) {
 				$value = (int)$this->mimetypeLoader->getId($value);
-			} elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) {
+			} elseif ($type === ISearchComparison::COMPARE_LIKE) {
 				// transform "mimetype='foo/%'" to "mimepart='foo'"
 				if (preg_match('|(.+)/%|', $value, $matches)) {
 					$field = 'mimepart';
@@ -183,6 +237,7 @@ private function getOperatorFieldAndValue(ISearchComparison $operator) {
 		} elseif ($field === 'favorite') {
 			$field = 'tag.category';
 			$value = self::TAG_FAVORITE;
+			$paramType = 'string';
 		} elseif ($field === 'name') {
 			$field = 'file.name';
 		} elseif ($field === 'tagname') {
@@ -191,53 +246,49 @@ private function getOperatorFieldAndValue(ISearchComparison $operator) {
 			$field = 'systemtag.name';
 		} elseif ($field === 'fileid') {
 			$field = 'file.fileid';
-		} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true)) {
+		} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) {
 			$field = 'path_hash';
 			$value = md5((string)$value);
 		} elseif ($field === 'owner') {
 			$field = 'uid_owner';
 		}
-		return [$field, $value, $type];
+		return [$field, $value, $type, $paramType];
 	}
 
 	private function validateComparison(ISearchComparison $operator) {
-		$types = [
-			'mimetype' => 'string',
-			'mtime' => 'integer',
-			'name' => 'string',
-			'path' => 'string',
-			'size' => 'integer',
-			'tagname' => 'string',
-			'systemtag' => 'string',
-			'favorite' => 'boolean',
-			'fileid' => 'integer',
-			'storage' => 'integer',
-			'share_with' => 'string',
-			'share_type' => 'integer',
-			'owner' => 'string',
-		];
 		$comparisons = [
-			'mimetype' => ['eq', 'like'],
+			'mimetype' => ['eq', 'like', 'in'],
 			'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'],
-			'name' => ['eq', 'like', 'clike'],
-			'path' => ['eq', 'like', 'clike'],
+			'name' => ['eq', 'like', 'clike', 'in'],
+			'path' => ['eq', 'like', 'clike', 'in'],
 			'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
 			'tagname' => ['eq', 'like'],
 			'systemtag' => ['eq', 'like'],
 			'favorite' => ['eq'],
-			'fileid' => ['eq'],
-			'storage' => ['eq'],
+			'fileid' => ['eq', 'in'],
+			'storage' => ['eq', 'in'],
 			'share_with' => ['eq'],
 			'share_type' => ['eq'],
 			'owner' => ['eq'],
 		];
 
-		if (!isset($types[$operator->getField()])) {
+		if (!isset(self::$fieldTypes[$operator->getField()])) {
 			throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField());
 		}
-		$type = $types[$operator->getField()];
-		if (gettype($operator->getValue()) !== $type) {
-			throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
+		$type = self::$fieldTypes[$operator->getField()];
+		if ($operator->getType() === ISearchComparison::COMPARE_IN) {
+			if (!is_array($operator->getValue())) {
+				throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
+			}
+			foreach ($operator->getValue() as $arrayValue) {
+				if (gettype($arrayValue) !== $type) {
+					throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField());
+				}
+			}
+		} else {
+			if (gettype($operator->getValue()) !== $type) {
+				throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
+			}
 		}
 		if (!in_array($operator->getType(), $comparisons[$operator->getField()])) {
 			throw new \InvalidArgumentException('Unsupported comparison for field  ' . $operator->getField() . ': ' . $operator->getType());
@@ -246,6 +297,7 @@ private function validateComparison(ISearchComparison $operator) {
 
 
 	private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array {
+		$paramType = self::$fieldTypes[$field];
 		$field = $operator->getField();
 		$value = $operator->getValue();
 		$type = $operator->getType();
@@ -259,17 +311,17 @@ private function getExtraOperatorField(ISearchComparison $operator, IMetadataQue
 				throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra());
 		}
 
-		return [$field, $value, $type];
+		return [$field, $value, $type, $paramType];
 	}
 
-	private function getParameterForValue(IQueryBuilder $builder, $value) {
+	private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) {
 		if ($value instanceof \DateTime) {
 			$value = $value->getTimestamp();
 		}
-		if (is_numeric($value)) {
-			$type = IQueryBuilder::PARAM_INT;
+		if (is_array($value)) {
+			$type = self::$paramArrayTypeMap[$paramType];
 		} else {
-			$type = IQueryBuilder::PARAM_STR;
+			$type = self::$paramTypeMap[$paramType];
 		}
 		return $builder->createNamedParameter($value, $type);
 	}

lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OC\Files\Search\SearchBinaryOperator;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchOperator;
+
+class FlattenNestedBool extends QueryOptimizerStep {
+	public function processOperator(ISearchOperator &$operator) {
+		if (
+			$operator instanceof SearchBinaryOperator && (
+				$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
+				$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
+			)
+		) {
+			$newArguments = [];
+			foreach ($operator->getArguments() as $oldArgument) {
+				if ($oldArgument instanceof SearchBinaryOperator && $oldArgument->getType() === $operator->getType()) {
+					$newArguments = array_merge($newArguments, $oldArgument->getArguments());
+				} else {
+					$newArguments[] = $oldArgument;
+				}
+			}
+			$operator->setArguments($newArguments);
+		}
+		parent::processOperator($operator);
+	}
+
+}

lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace OC\Files\Search\QueryOptimizer;
+
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchOperator;
+
+/**
+ * replace single argument AND and OR operations with their single argument
+ */
+class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep {
+	public function processOperator(ISearchOperator &$operator): bool {
+		parent::processOperator($operator);
+		if (
+			$operator instanceof ISearchBinaryOperator &&
+			count($operator->getArguments()) === 1 &&
+			(
+				$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
+				$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
+			)
+		) {
+			$operator = $operator->getArguments()[0];
+			return true;
+		}
+		return false;
+	}
+}