feat(preview): Implement scanning for previews

This work similarly to the move preview job to migrate the previews to
the new DB table and also reuse some code.

So when we are finding files in appdata/preview, try adding them to the
oc_previews table and delete them from the oc_filecache table.

Signed-off-by: Carl Schwan <[email protected]>

apps/files/lib/Command/ScanAppData.php

@@ -12,6 +12,7 @@
 use OC\DB\ConnectionAdapter;
 use OC\Files\Utils\Scanner;
 use OC\ForbiddenException;
+use OC\Preview\Storage\StorageFactory;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Folder;
 use OCP\Files\IRootFolder;
@@ -32,10 +33,12 @@ class ScanAppData extends Base {
 	protected int $foldersCounter = 0;
 
 	protected int $filesCounter = 0;
+	protected int $previewsCounter = -1;
 
 	public function __construct(
 		protected IRootFolder $rootFolder,
 		protected IConfig $config,
+		private StorageFactory $previewStorage,
 	) {
 		parent::__construct();
 	}
@@ -51,9 +54,12 @@ protected function configure(): void {
 	}
 
 	protected function scanFiles(OutputInterface $output, string $folder): int {
-		if ($folder === 'preview') {
-			$output->writeln('<error>Scanning the preview folder is not supported.</error>');
-			return self::FAILURE;
+		if ($folder === 'preview' || $folder === '') {
+			$this->previewsCounter = $this->previewStorage->scan();
+
+			if ($folder === 'preview') {
+				return self::SUCCESS;
+			}
 		}
 
 		try {
@@ -139,7 +145,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 		$this->initTools();
 
 		$exitCode = $this->scanFiles($output, $folder);
-		if ($exitCode === 0) {
+		if ($exitCode === self::SUCCESS) {
 			$this->presentStats($output);
 		}
 		return $exitCode;
@@ -167,7 +173,7 @@ protected function initTools(): void {
 	 *
 	 * @throws \ErrorException
 	 */
-	public function exceptionErrorHandler($severity, $message, $file, $line) {
+	public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void {
 		if (!(error_reporting() & $severity)) {
 			// This error code is not included in error_reporting
 			return;
@@ -178,10 +184,12 @@ public function exceptionErrorHandler($severity, $message, $file, $line) {
 	protected function presentStats(OutputInterface $output): void {
 		// Stop the timer
 		$this->execTime += microtime(true);
-
-		$headers = [
-			'Folders', 'Files', 'Elapsed time'
-		];
+		if ($this->previewsCounter !== -1) {
+			$headers[] = 'Previews';
+		}
+		$headers[] = 'Folders';
+		$headers[] = 'Files';
+		$headers[] = 'Elapsed time';
 
 		$this->showSummary($headers, null, $output);
 	}
@@ -192,14 +200,15 @@ protected function presentStats(OutputInterface $output): void {
 	 * @param string[] $headers
 	 * @param string[] $rows
 	 */
-	protected function showSummary($headers, $rows, OutputInterface $output): void {
+	protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void {
 		$niceDate = $this->formatExecTime();
 		if (!$rows) {
-			$rows = [
-				$this->foldersCounter,
-				$this->filesCounter,
-				$niceDate,
-			];
+			if ($this->previewsCounter !== -1) {
+				$rows[] = $this->previewsCounter;
+			}
+			$rows[] = $this->foldersCounter;
+			$rows[] = $this->filesCounter;
+			$rows[] = $niceDate;
 		}
 		$table = new Table($output);
 		$table

build/psalm-baseline.xml

@@ -1240,9 +1240,11 @@
       <code><![CDATA[listen]]></code>
       <code><![CDATA[listen]]></code>
     </DeprecatedMethod>
+    <InvalidArgument>
+      <code><![CDATA[[$this, 'exceptionErrorHandler']]]></code>
+    </InvalidArgument>
     <NullArgument>
       <code><![CDATA[null]]></code>
-      <code><![CDATA[null]]></code>
     </NullArgument>
   </file>
   <file src="apps/files/lib/Controller/DirectEditingController.php">

core/BackgroundJobs/MovePreviewJob.php

@@ -82,13 +82,18 @@ protected function run(mixed $argument): void {
 				->setMaxResults(100);
 
 			$result = $qb->executeQuery();
+			$foundOldPreview = false;
 			while ($row = $result->fetch()) {
 				$pathSplit = explode('/', $row['path']);
 				assert(count($pathSplit) >= 2);
 				$fileId = $pathSplit[count($pathSplit) - 2];
 				array_pop($pathSplit);
-				$path = implode('/', $pathSplit);
 				$this->processPreviews($fileId, true);
+				$foundOldPreview = true;
+			}
+
+			if (!$foundOldPreview) {
+				break;
 			}
 
 			// Stop if execution time is more than one hour.
@@ -114,44 +119,21 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
 		$folder = $this->appData->getFolder($internalPath);
 
 		/**
-		 * @var list<array{
-		 *     file: SimpleFile, width: int, height: int, crop: bool, max: bool, extension: string, mtime: int, size: int, version: ?int
-		 * }> $previewFiles
+		 * @var list<array{file: SimpleFile, preview: Preview}> $previewFiles
 		 */
 		$previewFiles = [];
 
 		foreach ($folder->getDirectoryListing() as $previewFile) {
 			/** @var SimpleFile $previewFile */
-			[0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName());
-			$nameSplit = explode('-', $baseName);
-
-			$offset = 0;
-			$version = null;
-			if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) {
-				$offset = 1;
-				$version = (int)$nameSplit[0];
-			}
-
-			$width = (int)$nameSplit[$offset + 0];
-			$height = (int)$nameSplit[$offset + 1];
-
-			$crop = false;
-			$max = false;
-			if (isset($nameSplit[$offset + 2])) {
-				$crop = $nameSplit[$offset + 2] === 'crop';
-				$max = $nameSplit[$offset + 2] === 'max';
-			}
+			$preview = Preview::fromPath($fileId . '/' . $previewFile->getName());
+			$preview->setSize($previewFile->getSize());
+			$preview->setMtime($previewFile->getMtime());
+			$preview->setOldFileId($previewFile->getId());
+			$preview->setEncrypted(false);
 
 			$previewFiles[] = [
 				'file' => $previewFile,
-				'width' => $width,
-				'height' => $height,
-				'crop' => $crop,
-				'version' => $version,
-				'max' => $max,
-				'extension' => $extension,
-				'size' => $previewFile->getSize(),
-				'mtime' => $previewFile->getMTime(),
+				'preview' => $preview,
 			];
 		}
 
@@ -166,27 +148,11 @@ private function processPreviews(int|string $fileId, bool $simplePaths): void {
 
 		if (count($result) > 0) {
 			foreach ($previewFiles as $previewFile) {
-				$preview = new Preview();
-				$preview->setFileId((int)$fileId);
+				$preview = $previewFile['preview'];
 				/** @var SimpleFile $file */
 				$file = $previewFile['file'];
-				$preview->setOldFileId($file->getId());
 				$preview->setStorageId($result[0]['storage']);
 				$preview->setEtag($result[0]['etag']);
-				$preview->setMtime($previewFile['mtime']);
-				$preview->setWidth($previewFile['width']);
-				$preview->setHeight($previewFile['height']);
-				$preview->setCropped($previewFile['crop']);
-				$preview->setVersion($previewFile['version']);
-				$preview->setMax($previewFile['max']);
-				$preview->setEncrypted(false);
-				$preview->setMimetype(match ($previewFile['extension']) {
-					'png' => IPreview::MIMETYPE_PNG,
-					'webp' => IPreview::MIMETYPE_WEBP,
-					'gif' => IPreview::MIMETYPE_GIF,
-					default => IPreview::MIMETYPE_JPEG,
-				});
-				$preview->setSize($previewFile['size']);
 				try {
 					$preview = $this->previewMapper->insert($preview);
 				} catch (Exception $e) {

lib/private/Files/Cache/Scanner.php

@@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner {
 
 	protected IDBConnection $connection;
 
+	private string $previewFolder;
+
 	public function __construct(\OC\Files\Storage\Storage $storage) {
 		$this->storage = $storage;
 		$this->storageId = $this->storage->getId();
@@ -75,6 +77,7 @@ public function __construct(\OC\Files\Storage\Storage $storage) {
 		$this->useTransactions = !$config->getValue('filescanner_no_transactions', false);
 		$this->lockingProvider = \OC::$server->get(ILockingProvider::class);
 		$this->connection = \OC::$server->get(IDBConnection::class);
+		$this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview';
 	}
 
 	/**
@@ -318,7 +321,6 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc
 
 		try {
 			$data = $this->scanFile($path, $reuse, -1, lock: $lock);
-
 			if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') {
 				$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']);
 				$data['size'] = $size;
@@ -413,6 +415,11 @@ protected function scanChildren(string $path, $recursive, int $reuse, int $folde
 		$size = 0;
 		$childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged);
 
+		if (str_starts_with($path, $this->previewFolder)) {
+			// Preview scanning is handled in LocalPreviewStorage
+			return 0;
+		}
+
 		foreach ($childQueue as $child => [$childId, $childSize]) {
 			// "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed
 			$childEtagChanged = false;

lib/private/Preview/Db/Preview.php

@@ -86,6 +86,41 @@ public function __construct() {
 		$this->addType('version', Types::BIGINT);
 	}
 
+	public static function fromPath(string $path): Preview {
+		$preview = new self();
+		$preview->setFileId((int)basename(dirname($path)));
+
+		$fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION);
+
+		[0 => $baseName, 1 => $extension] = explode('.', $fileName);
+		$preview->setMimetype(match ($extension) {
+			'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG,
+			'png' => IPreview::MIMETYPE_PNG,
+			'gif' => IPreview::MIMETYPE_GIF,
+			'webp' => IPreview::MIMETYPE_WEBP,
+			default => IPreview::MIMETYPE_JPEG,
+		});
+		$nameSplit = explode('-', $baseName);
+
+		$offset = 0;
+		$preview->setVersion(null);
+		if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) {
+			$offset = 1;
+			$preview->setVersion((int)$nameSplit[0]);
+		}
+
+		$preview->setWidth((int)$nameSplit[$offset + 0]);
+		$preview->setHeight((int)$nameSplit[$offset + 1]);
+
+		$preview->setCropped(false);
+		$preview->setMax(false);
+		if (isset($nameSplit[$offset + 2])) {
+			$preview->setCropped($nameSplit[$offset + 2] === 'crop');
+			$preview->setMax($nameSplit[$offset + 2] === 'max');
+		}
+		return $preview;
+	}
+
 	public function getName(): string {
 		$path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight();
 		if ($this->isCropped()) {

lib/private/Preview/Storage/IPreviewStorage.php

@@ -36,4 +36,6 @@ public function deletePreview(Preview $preview): void;
 	 * @throw \Exception
 	 */
 	public function migratePreview(Preview $preview, SimpleFile $file): void;
+
+	public function scan(): int;
 }