[stable9] Filecache repair step

apps/files/appinfo/app.php

@@ -41,6 +41,9 @@
 
 \OC::$server->getSearch()->registerProvider('OC\Search\Provider\File', array('apps' => array('files')));
 
+// instantiate to make sure services get registered
+$app = new \OCA\Files\AppInfo\Application();
+
 $templateManager = \OC_Helper::getFileTemplateManager();
 $templateManager->registerTemplate('text/html', 'core/templates/filetemplates/template.html');
 $templateManager->registerTemplate('application/vnd.oasis.opendocument.presentation', 'core/templates/filetemplates/template.odp');

apps/files/appinfo/application.php

@@ -69,6 +69,13 @@ public function __construct(array $urlParams=array()) {
 			);
 		});
 
+		$container->registerService('OCP\Lock\ILockingProvider', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getLockingProvider();
+		});
+		$container->registerService('OCP\Files\IMimeTypeLoader', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getMimeTypeLoader();
+		});
+
 		/*
 		 * Register capabilities
 		 */

apps/files/appinfo/register_command.php

@@ -25,8 +25,11 @@
 $userManager = OC::$server->getUserManager();
 $shareManager = \OC::$server->getShareManager();
 $mountManager = \OC::$server->getMountManager();
+$lockingProvider = \OC::$server->getLockingProvider();
+$mimeTypeLoader = \OC::$server->getMimeTypeLoader();
+$config = \OC::$server->getConfig();
 
 /** @var Symfony\Component\Console\Application $application */
-$application->add(new OCA\Files\Command\Scan($userManager));
+$application->add(new OCA\Files\Command\Scan($userManager, $lockingProvider, $mimeTypeLoader, $config));
 $application->add(new OCA\Files\Command\DeleteOrphanedFiles($dbConnection));
 $application->add(new OCA\Files\Command\TransferOwnership($userManager, $shareManager, $mountManager));

apps/files/command/scan.php

@@ -38,20 +38,39 @@
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Helper\Table;
+use OC\Repair\RepairMismatchFileCachePath;
+use OCP\Lock\ILockingProvider;
+use OCP\Lock\LockedException;
+use OCP\Files\IMimeTypeLoader;
+use OCP\IConfig;
 
 class Scan extends Base {
 
 	/** @var IUserManager $userManager */
 	private $userManager;
+	/** @var ILockingProvider */
+	private $lockingProvider;
+	/** @var IMimeTypeLoader */
+	private $mimeTypeLoader;
+	/** @var IConfig */
+	private $config;
 	/** @var float */
 	protected $execTime = 0;
 	/** @var int */
 	protected $foldersCounter = 0;
 	/** @var int */
 	protected $filesCounter = 0;
 
-	public function __construct(IUserManager $userManager) {
+	public function __construct(
+		IUserManager $userManager,
+		ILockingProvider $lockingProvider,
+ 		IMimeTypeLoader $mimeTypeLoader,
+		IConfig $config
+	) {
 		$this->userManager = $userManager;
+		$this->lockingProvider = $lockingProvider;
+		$this->mimeTypeLoader = $mimeTypeLoader;
+		$this->config = $config;
 		parent::__construct();
 	}
 
@@ -89,12 +108,65 @@ protected function configure() {
 				null,
 				InputOption::VALUE_NONE,
 				'will rescan all files of all known users'
+			)
+			->addOption(
+				'repair',
+				null,
+				InputOption::VALUE_NONE,
+				'will repair detached filecache entries (slow)'
 			);
 	}
 
-	protected function scanFiles($user, $path, $verbose, OutputInterface $output) {
+	/**
+	 * Repair all storages at once
+	 *
+	 * @param OutputInterface $output
+	 */
+	protected function repairAll(OutputInterface $output) {
+		$connection = $this->reconnectToDatabase($output);
+		$repairStep = new RepairMismatchFileCachePath(
+			$connection,
+			$this->mimeTypeLoader
+		);
+		$repairStep->setStorageNumericId(null);
+		$repairStep->setCountOnly(false);
+		$repairStep->run(new ConsoleOutput($output));
+	}
+
+	protected function scanFiles($user, $path, $verbose, OutputInterface $output, $shouldRepair = false) {
 		$connection = $this->reconnectToDatabase($output);
 		$scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->getLogger());
+
+		if ($shouldRepair) {
+			$scanner->listen('\OC\Files\Utils\Scanner', 'beforeScanStorage', function ($storage) use ($output, $connection) {
+				try {
+					// FIXME: this will lock the storage even if there is nothing to repair
+					$storage->acquireLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				} catch (LockedException $e) {
+					$output->writeln("\t<error>Storage \"" . $storage->getCache()->getNumericStorageId() . '" cannot be repaired as it is currently in use, please try again later</error>');
+					return;
+				}
+				$stored = false;
+				try {
+					$repairStep = new RepairMismatchFileCachePath(
+						$connection,
+						$this->mimeTypeLoader
+					);
+					$repairStep->setStorageNumericId($storage->getCache()->getNumericStorageId());
+					$repairStep->setCountOnly(false);
+					$repairStep->run();
+				} catch (\Exception $e) {
+					$stored = $e;
+				}
+
+				// Release the lock first
+				$storage->releaseLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				// Now throw the exception for handling elsewhere
+				if($stored) {
+					throw $stored;
+				}
+			});
+		}
 		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
 		# printout and count
 		if ($verbose) {
@@ -132,7 +204,7 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output) {
 		}
 
 		try {
-			$scanner->scan($path);
+			$scanner->scan($path, $shouldRepair);
 		} catch (ForbiddenException $e) {
 			$output->writeln("<error>Home storage for user $user not writable</error>");
 			$output->writeln("Make sure you're running the scan command only as the user the web server runs as");
@@ -148,12 +220,26 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output) {
 
 	protected function execute(InputInterface $input, OutputInterface $output) {
 		$inputPath = $input->getOption('path');
+
+		$shouldRepairStoragesIndividually = (bool) $input->getOption('repair');
+
 		if ($inputPath) {
 			$inputPath = '/' . trim($inputPath, '/');
 			list (, $user,) = explode('/', $inputPath, 3);
 			$users = array($user);
 		} else if ($input->getOption('all')) {
-			$users = $this->userManager->search('');
+			// we can only repair all storages in bulk (more efficient) if singleuser or maintenance mode
+			// is enabled to prevent concurrent user access
+			if ($input->getOption('repair') && ($this->config->getSystemValue('singleuser', false) || $this->config->getSystemValue('maintenance', false))) {
+				// repair all storages at once
+				$this->repairAll($output);
+				// don't fix individually
+				$shouldRepairStoragesIndividually = false;
+			} else {
+				$output->writeln("<comment>Repairing every storage individually is slower than repairing in bulk</comment>");
+				$output->writeln("<comment>To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on</comment>");
+				$users = $this->userManager->search('');
+			}
 		} else {
 			$users = $input->getArgument('user_id');
 		}
@@ -198,9 +284,10 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			if ($this->userManager->userExists($user)) {
 				# add an extra line when verbose is set to optical separate users
 				if ($verbose) {$output->writeln(""); }
-				$output->writeln("Starting scan for user $user_count out of $users_total ($user)");
+				$r = $shouldRepairStoragesIndividually ? ' (and repair)' : '';
+				$output->writeln("Starting scan$r for user $user_count out of $users_total ($user)");
 				# full: printout data if $verbose was set
-				$this->scanFiles($user, $path, $verbose, $output);
+				$this->scanFiles($user, $path, $verbose, $output, $shouldRepairStoragesIndividually);
 			} else {
 				$output->writeln("<error>Unknown user $user_count $user</error>");
 			}

lib/private/files/utils/scanner.php

@@ -166,6 +166,9 @@ public function scan($dir = '') {
 			if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) {
 				continue;
 			}
+
+			$this->emit('\OC\Files\Utils\Scanner', 'beforeScanStorage', [$storage]);
+
 			$relativePath = $mount->getInternalPath($dir);
 			$scanner = $storage->getScanner();
 			$scanner->setUseTransactions(false);
@@ -200,6 +203,9 @@ public function scan($dir = '') {
 			if (!$isDbLocking) {
 				$this->db->commit();
 			}
+
+			$this->emit('\OC\Files\Utils\Scanner', 'afterScanStorage', [$storage]);
+
 		}
 	}
 

lib/private/repair.php

@@ -39,6 +39,7 @@
 use OC\Repair\EncryptionCompatibility;
 use OC\Repair\OldGroupMembershipShares;
 use OC\Repair\RemoveGetETagEntries;
+use OC\Repair\RepairMismatchFileCachePath;
 use OC\Repair\SharePropagation;
 use OC\Repair\SqliteAutoincrement;
 use OC\Repair\DropOldTables;
@@ -109,6 +110,7 @@ public function addStep($repairStep) {
 	public static function getRepairSteps() {
 		return [
 			new RepairMimeTypes(\OC::$server->getConfig()),
+			new RepairMismatchFileCachePath(\OC::$server->getDatabaseConnection(), \OC::$server->getMimeTypeLoader()),
 			new AssetCache(),
 			new FillETags(\OC::$server->getDatabaseConnection()),
 			new CleanTags(\OC::$server->getDatabaseConnection()),