diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index d8156da3669e3..a340433fcfb5d 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -107,15 +107,6 @@ protected function configure() { ); } - public function checkScanWarning($fullPath, OutputInterface $output) { - $normalizedPath = basename(\OC\Files\Filesystem::normalizePath($fullPath)); - $path = basename($fullPath); - - if ($normalizedPath !== $path) { - $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); - } - } - protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) { $connection = $this->reconnectToDatabase($output); $scanner = new \OC\Files\Utils\Scanner( @@ -143,12 +134,8 @@ protected function scanFiles($user, $path, OutputInterface $output, $backgroundS $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); }); - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { - $this->checkScanWarning($path, $output); - }); - - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { - $this->checkScanWarning($path, $output); + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output) { + $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); }); try { diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 854445c82dcd2..5517e6b93e3fb 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -74,15 +74,6 @@ protected function configure() { $this->addArgument('folder', InputArgument::OPTIONAL, 'The appdata subfolder to scan', ''); } - public function checkScanWarning($fullPath, OutputInterface $output) { - $normalizedPath = basename(\OC\Files\Filesystem::normalizePath($fullPath)); - $path = basename($fullPath); - - if ($normalizedPath !== $path) { - $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); - } - } - protected function scanFiles(OutputInterface $output, string $folder): int { try { $appData = $this->getAppDataFolder(); @@ -125,12 +116,8 @@ protected function scanFiles(OutputInterface $output, string $folder): int { $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); }); - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { - $this->checkScanWarning($path, $output); - }); - - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { - $this->checkScanWarning($path, $output); + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output) { + $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); }); try { diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index cea7158b1b6f4..6edb74a7e26cb 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1131,6 +1131,7 @@ 'OC\\Files\\Storage\\Temporary' => $baseDir . '/lib/private/Files/Storage/Temporary.php', 'OC\\Files\\Storage\\Wrapper\\Availability' => $baseDir . '/lib/private/Files/Storage/Wrapper/Availability.php', 'OC\\Files\\Storage\\Wrapper\\Encoding' => $baseDir . '/lib/private/Files/Storage/Wrapper/Encoding.php', + 'OC\\Files\\Storage\\Wrapper\\EncodingDirectoryWrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php', 'OC\\Files\\Storage\\Wrapper\\Encryption' => $baseDir . '/lib/private/Files/Storage/Wrapper/Encryption.php', 'OC\\Files\\Storage\\Wrapper\\Jail' => $baseDir . '/lib/private/Files/Storage/Wrapper/Jail.php', 'OC\\Files\\Storage\\Wrapper\\PermissionsMask' => $baseDir . '/lib/private/Files/Storage/Wrapper/PermissionsMask.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 1bfaea25c688b..c898ec89276c4 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1160,6 +1160,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Storage\\Temporary' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Temporary.php', 'OC\\Files\\Storage\\Wrapper\\Availability' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Availability.php', 'OC\\Files\\Storage\\Wrapper\\Encoding' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Encoding.php', + 'OC\\Files\\Storage\\Wrapper\\EncodingDirectoryWrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php', 'OC\\Files\\Storage\\Wrapper\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Encryption.php', 'OC\\Files\\Storage\\Wrapper\\Jail' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Jail.php', 'OC\\Files\\Storage\\Wrapper\\PermissionsMask' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/PermissionsMask.php', diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 7bb0341e20105..c0d5e5a89e6d0 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -38,6 +38,7 @@ use Doctrine\DBAL\Exception; use OC\Files\Filesystem; +use OC\Files\Storage\Wrapper\Encoding; use OC\Hooks\BasicEmitter; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; @@ -420,7 +421,16 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s if ($permissions === 0) { continue; } - $file = $fileMeta['name']; + $originalFile = $fileMeta['name']; + $file = trim(\OC\Files\Filesystem::normalizePath($originalFile), '/'); + if (trim($originalFile, '/') !== $file) { + // encoding mismatch, might require compatibility wrapper + \OC::$server->getLogger()->debug('Scanner: Skipping non-normalized file name "'. $originalFile . '" in path "' . $path . '".', ['app' => 'core']); + $this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]); + // skip this entry + continue; + } + $newChildNames[] = $file; $child = $path ? $path . '/' . $file : $file; try { diff --git a/lib/private/Files/Storage/Wrapper/Encoding.php b/lib/private/Files/Storage/Wrapper/Encoding.php index e2b486546afa9..252ef1b499dbe 100644 --- a/lib/private/Files/Storage/Wrapper/Encoding.php +++ b/lib/private/Files/Storage/Wrapper/Encoding.php @@ -29,6 +29,7 @@ namespace OC\Files\Storage\Wrapper; use OC\Cache\CappedMemoryCache; +use OC\Files\Filesystem; use OCP\Files\Storage\IStorage; use OCP\ICache; @@ -162,7 +163,8 @@ public function rmdir($path) { * @return resource|bool */ public function opendir($path) { - return $this->storage->opendir($this->findPathToUse($path)); + $handle = $this->storage->opendir($this->findPathToUse($path)); + return EncodingDirectoryWrapper::wrap($handle); } /** @@ -532,10 +534,16 @@ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t } public function getMetaData($path) { - return $this->storage->getMetaData($this->findPathToUse($path)); + $entry = $this->storage->getMetaData($this->findPathToUse($path)); + $entry['name'] = trim(Filesystem::normalizePath($entry['name']), '/'); + return $entry; } public function getDirectoryContent($directory): \Traversable { - return $this->storage->getDirectoryContent($this->findPathToUse($directory)); + $entries = $this->storage->getDirectoryContent($this->findPathToUse($directory)); + foreach ($entries as $entry) { + $entry['name'] = trim(Filesystem::normalizePath($entry['name']), '/'); + yield $entry; + } } } diff --git a/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php b/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php new file mode 100644 index 0000000000000..935a15af4cfbc --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/EncodingDirectoryWrapper.php @@ -0,0 +1,55 @@ + + * @author Vincent Petry + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Files\Storage\Wrapper; + +use Icewind\Streams\DirectoryWrapper; +use OC\Files\Filesystem; + +/** + * Normalize file names while reading directory entries + */ +class EncodingDirectoryWrapper extends DirectoryWrapper { + /** + * @return string + */ + public function dir_readdir() { + $file = readdir($this->source); + if ($file !== false && $file !== '.' && $file !== '..') { + $file = trim(Filesystem::normalizePath($file), '/'); + } + + return $file; + } + + /** + * @param resource $source + * @param callable $filter + * @return resource|bool + */ + public static function wrap($source) { + return self::wrapSource($source, [ + 'source' => $source, + ]); + } +} diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 5509a49941b19..b886eaca80d33 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -146,6 +146,9 @@ protected function attachListener($mount) { $this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', [$mount->getMountPoint() . $path]); $this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path)); }); + $scanner->listen('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', function ($path) use ($mount) { + $this->emit('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', [$path]); + }); } /** diff --git a/tests/lib/Files/Storage/Wrapper/EncodingTest.php b/tests/lib/Files/Storage/Wrapper/EncodingTest.php index 498d9f7824811..0901edf83fa06 100644 --- a/tests/lib/Files/Storage/Wrapper/EncodingTest.php +++ b/tests/lib/Files/Storage/Wrapper/EncodingTest.php @@ -32,7 +32,7 @@ protected function tearDown(): void { public function directoryProvider() { $a = parent::directoryProvider(); - $a[] = [self::NFD_NAME]; + $a[] = [self::NFC_NAME]; return $a; } @@ -199,4 +199,46 @@ public function testCopyAndMoveFromStorageEncodedFolder($sourceDir, $targetDir) $this->assertEquals('bar', $this->instance->file_get_contents(self::NFC_NAME . '2/test2.txt')); } + + public function testNormalizedDirectoryEntriesOpenDir() { + $this->sourceStorage->mkdir('/test'); + $this->sourceStorage->mkdir('/test/' . self::NFD_NAME); + + $this->assertTrue($this->instance->file_exists('/test/' . self::NFC_NAME)); + $this->assertTrue($this->instance->file_exists('/test/' . self::NFD_NAME)); + + $dh = $this->instance->opendir('/test'); + $content = []; + while ($file = readdir($dh)) { + if ($file != '.' and $file != '..') { + $content[] = $file; + } + } + + $this->assertCount(1, $content); + $this->assertEquals(self::NFC_NAME, $content[0]); + } + + public function testNormalizedDirectoryEntriesGetDirectoryContent() { + $this->sourceStorage->mkdir('/test'); + $this->sourceStorage->mkdir('/test/' . self::NFD_NAME); + + $this->assertTrue($this->instance->file_exists('/test/' . self::NFC_NAME)); + $this->assertTrue($this->instance->file_exists('/test/' . self::NFD_NAME)); + + $content = iterator_to_array($this->instance->getDirectoryContent('/test')); + $this->assertCount(1, $content); + $this->assertEquals(self::NFC_NAME, $content[0]['name']); + } + + public function testNormalizedGetMetaData() { + $this->sourceStorage->mkdir('/test'); + $this->sourceStorage->mkdir('/test/' . self::NFD_NAME); + + $entry = $this->instance->getMetaData('/test/' . self::NFC_NAME); + $this->assertEquals(self::NFC_NAME, $entry['name']); + + $entry = $this->instance->getMetaData('/test/' . self::NFD_NAME); + $this->assertEquals(self::NFC_NAME, $entry['name']); + } }