fix: make background scan job compatible with sharding

Signed-off-by: Robin Appelman <[email protected]>

apps/files/lib/BackgroundJob/ScanFiles.php

@@ -33,11 +33,11 @@ class ScanFiles extends TimedJob {
 	public const USERS_PER_SESSION = 500;
 
 	public function __construct(
-		IConfig $config,
+		IConfig          $config,
 		IEventDispatcher $dispatcher,
-		LoggerInterface $logger,
-		IDBConnection $connection,
-		ITimeFactory $time
+		LoggerInterface  $logger,
+		IDBConnection    $connection,
+		ITimeFactory     $time
 	) {
 		parent::__construct($time);
 		// Run once per 10 minutes
@@ -70,15 +70,61 @@ protected function runScanner(string $user): void {
 	 * @return string|false
 	 */
 	private function getUserToScan() {
+		if ($this->connection->getShardDefinition("filecache")) {
+			// for sharded filecache, the "LIMIT" from the normal query doesn't work
+
+			// first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted.
+			// we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->setMaxResults(10)
+				->groupBy("f.storage")
+				->runAcrossAllShards();
+
+			$result = $query->executeQuery();
+			while ($res = $result->fetch()) {
+				if ($res['user_id']) {
+					return $res['user_id'];
+				}
+			}
+
+			// as a fallback, we try a slower approach where we find all mounted storages first
+			// this is essentially doing the inner join manually
+			$storages = $this->getAllMountedStorages();
+
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY)))
+				->setMaxResults(1)
+				->runAcrossAllShards();
+			return $query->executeQuery()->fetchOne();
+		} else {
+			$query = $this->connection->getQueryBuilder();
+			$query->select('m.user_id')
+				->from('filecache', 'f')
+				->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
+				->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
+				->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
+				->setMaxResults(1)
+				->runAcrossAllShards();
+
+			return $query->executeQuery()->fetchOne();
+		}
+	}
+
+	private function getAllMountedStorages(): array {
 		$query = $this->connection->getQueryBuilder();
-		$query->select('user_id')
-			->from('filecache', 'f')
-			->innerJoin('f', 'mounts', 'm', $query->expr()->eq('storage_id', 'storage'))
-			->where($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
-			->andWhere($query->expr()->gt('parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
-			->setMaxResults(1);
-
-		return $query->executeQuery()->fetchOne();
+		$query->selectDistinct('storage_id')
+			->from('mounts');
+		return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
 	}
 
 	/**