Skip to content

Commit 5d2198b

Browse files
authored
Merge pull request #25109 from nextcloud/external-scan
add command to scan external storages directly
2 parents b7088e7 + 42e14cc commit 5d2198b

File tree

6 files changed

+295
-74
lines changed

6 files changed

+295
-74
lines changed

apps/files_external/appinfo/info.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
4747
<command>OCA\Files_External\Command\Backends</command>
4848
<command>OCA\Files_External\Command\Verify</command>
4949
<command>OCA\Files_External\Command\Notify</command>
50+
<command>OCA\Files_External\Command\Scan</command>
5051
</commands>
5152

5253
<settings>

apps/files_external/composer/composer/autoload_classmap.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
2020
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
2121
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
22+
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
23+
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
2224
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
2325
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
2426
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',

apps/files_external/composer/composer/autoload_static.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
3434
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
3535
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
3636
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
37+
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
38+
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
3739
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
3840
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
3941
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',

apps/files_external/lib/Command/Notify.php

Lines changed: 6 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,13 @@
3030
namespace OCA\Files_External\Command;
3131

3232
use Doctrine\DBAL\Exception\DriverException;
33-
use OC\Core\Command\Base;
34-
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
35-
use OCA\Files_External\Lib\StorageConfig;
3633
use OCA\Files_External\Service\GlobalStoragesService;
3734
use OCP\DB\QueryBuilder\IQueryBuilder;
3835
use OCP\Files\Notify\IChange;
3936
use OCP\Files\Notify\INotifyHandler;
4037
use OCP\Files\Notify\IRenameChange;
4138
use OCP\Files\Storage\INotifyStorage;
4239
use OCP\Files\Storage\IStorage;
43-
use OCP\Files\StorageNotAvailableException;
4440
use OCP\IDBConnection;
4541
use OCP\IUserManager;
4642
use Psr\Log\LoggerInterface;
@@ -49,14 +45,14 @@
4945
use Symfony\Component\Console\Input\InputOption;
5046
use Symfony\Component\Console\Output\OutputInterface;
5147

52-
class Notify extends Base {
48+
class Notify extends StorageAuthBase {
5349
public function __construct(
54-
private GlobalStoragesService $globalService,
5550
private IDBConnection $connection,
5651
private LoggerInterface $logger,
57-
private IUserManager $userManager
52+
GlobalStoragesService $globalService,
53+
IUserManager $userManager,
5854
) {
59-
parent::__construct();
55+
parent::__construct($globalService, $userManager);
6056
}
6157

6258
protected function configure(): void {
@@ -97,71 +93,12 @@ protected function configure(): void {
9793
parent::configure();
9894
}
9995

100-
private function getUserOption(InputInterface $input): ?string {
101-
if ($input->getOption('user')) {
102-
return (string)$input->getOption('user');
103-
}
104-
105-
return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
106-
}
107-
108-
private function getPasswordOption(InputInterface $input): ?string {
109-
if ($input->getOption('password')) {
110-
return (string)$input->getOption('password');
111-
}
112-
113-
return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
114-
}
115-
11696
protected function execute(InputInterface $input, OutputInterface $output): int {
117-
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
118-
if (is_null($mount)) {
119-
$output->writeln('<error>Mount not found</error>');
97+
[$mount, $storage] = $this->createStorage($input, $output);
98+
if ($storage === null) {
12099
return self::FAILURE;
121100
}
122-
$noAuth = false;
123-
124-
$userOption = $this->getUserOption($input);
125-
$passwordOption = $this->getPasswordOption($input);
126-
127-
// if only the user is provided, we get the user object to pass along to the auth backend
128-
// this allows using saved user credentials
129-
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;
130-
131-
try {
132-
$authBackend = $mount->getAuthMechanism();
133-
$authBackend->manipulateStorageConfig($mount, $user);
134-
} catch (InsufficientDataForMeaningfulAnswerException $e) {
135-
$noAuth = true;
136-
} catch (StorageNotAvailableException $e) {
137-
$noAuth = true;
138-
}
139101

140-
if ($userOption) {
141-
$mount->setBackendOption('user', $userOption);
142-
}
143-
if ($passwordOption) {
144-
$mount->setBackendOption('password', $passwordOption);
145-
}
146-
147-
try {
148-
$backend = $mount->getBackend();
149-
$backend->manipulateStorageConfig($mount, $user);
150-
} catch (InsufficientDataForMeaningfulAnswerException $e) {
151-
$noAuth = true;
152-
} catch (StorageNotAvailableException $e) {
153-
$noAuth = true;
154-
}
155-
156-
try {
157-
$storage = $this->createStorage($mount);
158-
} catch (\Exception $e) {
159-
$output->writeln('<error>Error while trying to create storage</error>');
160-
if ($noAuth) {
161-
$output->writeln('<error>Login and/or password required</error>');
162-
}
163-
return self::FAILURE;
164-
}
165102
if (!$storage instanceof INotifyStorage) {
166103
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
167104
return self::FAILURE;
@@ -189,11 +126,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
189126
return self::SUCCESS;
190127
}
191128

192-
private function createStorage(StorageConfig $mount): IStorage {
193-
$class = $mount->getBackend()->getStorageClass();
194-
return new $class($mount->getBackendOptions());
195-
}
196-
197129
private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void {
198130
$parent = ltrim(dirname($path), '/');
199131
if ($parent === '.') {
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2021 Robin Appelman <[email protected]>
6+
*
7+
* @license GNU AGPL version 3 or any later version
8+
*
9+
* This program is free software: you can redistribute it and/or modify
10+
* it under the terms of the GNU Affero General Public License as
11+
* published by the Free Software Foundation, either version 3 of the
12+
* License, or (at your option) any later version.
13+
*
14+
* This program is distributed in the hope that it will be useful,
15+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
* GNU Affero General Public License for more details.
18+
*
19+
* You should have received a copy of the GNU Affero General Public License
20+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
21+
*
22+
*/
23+
24+
namespace OCA\Files_External\Command;
25+
26+
use OC\Files\Cache\Scanner;
27+
use OCA\Files_External\Service\GlobalStoragesService;
28+
use OCP\IUserManager;
29+
use Symfony\Component\Console\Helper\Table;
30+
use Symfony\Component\Console\Input\InputArgument;
31+
use Symfony\Component\Console\Input\InputInterface;
32+
use Symfony\Component\Console\Input\InputOption;
33+
use Symfony\Component\Console\Output\OutputInterface;
34+
35+
class Scan extends StorageAuthBase {
36+
protected float $execTime = 0;
37+
protected int $foldersCounter = 0;
38+
protected int $filesCounter = 0;
39+
40+
public function __construct(
41+
GlobalStoragesService $globalService,
42+
IUserManager $userManager
43+
) {
44+
parent::__construct($globalService, $userManager);
45+
}
46+
47+
protected function configure(): void {
48+
$this
49+
->setName('files_external:scan')
50+
->setDescription('Scan an external storage for changed files')
51+
->addArgument(
52+
'mount_id',
53+
InputArgument::REQUIRED,
54+
'the mount id of the mount to scan'
55+
)->addOption(
56+
'user',
57+
'u',
58+
InputOption::VALUE_REQUIRED,
59+
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
60+
)->addOption(
61+
'password',
62+
'p',
63+
InputOption::VALUE_REQUIRED,
64+
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
65+
)->addOption(
66+
'path',
67+
'',
68+
InputOption::VALUE_OPTIONAL,
69+
'The path in the storage to scan',
70+
''
71+
);
72+
parent::configure();
73+
}
74+
75+
protected function execute(InputInterface $input, OutputInterface $output): int {
76+
[, $storage] = $this->createStorage($input, $output);
77+
if ($storage === null) {
78+
return 1;
79+
}
80+
81+
$path = $input->getOption('path');
82+
83+
$this->execTime = -microtime(true);
84+
85+
/** @var Scanner $scanner */
86+
$scanner = $storage->getScanner();
87+
88+
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
89+
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
90+
++$this->filesCounter;
91+
$this->abortIfInterrupted();
92+
});
93+
94+
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
95+
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
96+
++$this->foldersCounter;
97+
$this->abortIfInterrupted();
98+
});
99+
100+
$scanner->scan($path);
101+
102+
$this->presentStats($output);
103+
104+
return 0;
105+
}
106+
107+
/**
108+
* @param OutputInterface $output
109+
*/
110+
protected function presentStats(OutputInterface $output): void {
111+
// Stop the timer
112+
$this->execTime += microtime(true);
113+
114+
$headers = [
115+
'Folders', 'Files', 'Elapsed time'
116+
];
117+
118+
$this->showSummary($headers, [], $output);
119+
}
120+
121+
/**
122+
* Shows a summary of operations
123+
*
124+
* @param string[] $headers
125+
* @param string[] $rows
126+
* @param OutputInterface $output
127+
*/
128+
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
129+
$niceDate = $this->formatExecTime();
130+
if (!$rows) {
131+
$rows = [
132+
$this->foldersCounter,
133+
$this->filesCounter,
134+
$niceDate,
135+
];
136+
}
137+
$table = new Table($output);
138+
$table
139+
->setHeaders($headers)
140+
->setRows([$rows]);
141+
$table->render();
142+
}
143+
144+
145+
/**
146+
* Formats microtime into a human readable format
147+
*
148+
* @return string
149+
*/
150+
protected function formatExecTime(): string {
151+
$secs = round($this->execTime);
152+
# convert seconds into HH:MM:SS form
153+
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
154+
}
155+
}

0 commit comments

Comments
 (0)