Merge pull request #37066 from nextcloud/backport/34835/stable26

[stable26] Run WebDAV integration tests against S3

Author: blizzz

.github/workflows/s3-primary-integration.yml

@@ -0,0 +1,84 @@
+name: S3 primary storage integration tests
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - stable*
+
+jobs:
+  s3-primary-integration-tests-minio:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      # do not stop on another job's failure
+      fail-fast: false
+      matrix:
+        php-versions: ['8.0']
+        key: ['objectstore', 'objectstore_multibucket']
+
+    name: php${{ matrix.php-versions }}-${{ matrix.key }}-minio
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - "6379:6379"
+      minio:
+        env:
+          MINIO_ACCESS_KEY: minio
+          MINIO_SECRET_KEY: minio123
+        image: bitnami/minio:2021.12.29
+        ports:
+          - "9000:9000"
+
+    steps:
+      - name: Checkout server
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Set up php ${{ matrix.php-versions }}
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-versions }}
+          tools: phpunit:9
+          extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd, redis
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Wait for S3
+        run: |
+          sleep 10
+          curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready
+
+      - name: Set up Nextcloud
+        run: |
+          mkdir data
+          echo '<?php $CONFIG=["${{ matrix.key }}" => ["class" => "OC\Files\ObjectStore\S3", "arguments" => ["bucket" => "nextcloud", "autocreate" => true, "key" => "minio", "secret" => "minio123", "hostname" => "localhost", "port" => 9000, "use_ssl" => false, "use_path_style" => true, "uploadPartSize" => 52428800]]];' > config/config.php
+          echo '<?php $CONFIG=["redis" => ["host" => "localhost", "port" => 6379], "memcache.local" => "\OC\Memcache\Redis", "memcache.distributed" => "\OC\Memcache\Redis"];' > config/redis.config.php
+          ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
+          php -f index.php
+
+      - name: Integration
+        run: |
+          cd build/integration
+          bash run.sh --tags "~@failure-s3" features/webdav-related.feature
+
+      - name: S3 logs
+        if: always()
+        run: |
+          cat data/nextcloud.log
+          docker ps -a
+          docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done
+
+
+  s3-primary-integration-summary:
+    runs-on: ubuntu-latest
+    needs: [s3-primary-integration-tests-minio]
+
+    if: always()
+
+    steps:
+      - name: Summary status
+        run: if ${{ needs.s3-primary-integration-tests-minio.result != 'success' }}; then exit 1; fi

apps/dav/lib/Connector/Sabre/Directory.php

@@ -35,7 +35,6 @@
 use OC\Files\Mount\MoveableMount;
 use OC\Files\View;
 use OC\Metadata\FileMetadata;
-use OC\Metadata\MetadataGroup;
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
@@ -57,7 +56,6 @@
 use OCP\Share\IManager as IShareManager;
 
 class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICollection, \Sabre\DAV\IQuota, \Sabre\DAV\IMoveTarget, \Sabre\DAV\ICopyTarget {
-
 	/**
 	 * Cached directory content
 	 * @var \OCP\Files\FileInfo[]
@@ -116,7 +114,6 @@ public function createFile($name, $data = null) {
 			// for chunked upload also updating a existing file is a "createFile"
 			// because we create all the chunks before re-assemble them to the existing file.
 			if (isset($_SERVER['HTTP_OC_CHUNKED'])) {
-
 				// exit if we can't create a new file and we don't updatable existing file
 				$chunkInfo = \OC_FileChunking::decodeName($name);
 				if (!$this->fileView->isCreatable($this->path) &&
@@ -328,8 +325,14 @@ public function getQuotaInfo() {
 		if ($this->quotaInfo) {
 			return $this->quotaInfo;
 		}
+		$relativePath = $this->fileView->getRelativePath($this->info->getPath());
+		if ($relativePath === null) {
+			$logger->warning("error while getting quota as the relative path cannot be found");
+			return [0, 0];
+		}
+
 		try {
-			$storageInfo = \OC_Helper::getStorageInfo($this->info->getPath(), $this->info, false);
+			$storageInfo = \OC_Helper::getStorageInfo($relativePath, $this->info, false);
 			if ($storageInfo['quota'] === \OCP\Files\FileInfo::SPACE_UNLIMITED) {
 				$free = \OCP\Files\FileInfo::SPACE_UNLIMITED;
 			} else {

apps/dav/tests/unit/Connector/Sabre/DirectoryTest.php

@@ -304,6 +304,10 @@ public function testGetQuotaInfoUnlimited(): void {
 			->method('free_space')
 			->willReturn(800);
 
+		$this->info->expects($this->any())
+			->method('getPath')
+			->willReturn('/admin/files/foo');
+
 		$this->info->expects($this->once())
 			->method('getSize')
 			->willReturn(200);
@@ -312,6 +316,10 @@ public function testGetQuotaInfoUnlimited(): void {
 			->method('getMountPoint')
 			->willReturn($mountPoint);
 
+		$this->view->expects($this->any())
+			->method('getRelativePath')
+			->willReturn('/foo');
+
 		$mountPoint->method('getMountPoint')
 			->willReturn('/user/files/mymountpoint');
 
@@ -359,6 +367,10 @@ public function testGetQuotaInfoSpecific(): void {
 		$mountPoint->method('getMountPoint')
 			->willReturn('/user/files/mymountpoint');
 
+		$this->view->expects($this->any())
+			->method('getRelativePath')
+			->willReturn('/foo');
+
 		$dir = new Directory($this->view, $this->info);
 		$this->assertEquals([200, 800], $dir->getQuotaInfo()); //200 used, 800 free
 	}

apps/provisioning_api/lib/Controller/UsersController.php

@@ -77,7 +77,6 @@
 use Psr\Log\LoggerInterface;
 
 class UsersController extends AUserData {
-
 	/** @var IURLGenerator */
 	protected $urlGenerator;
 	/** @var LoggerInterface */
@@ -374,7 +373,7 @@ public function addUser(
 				$group = $this->groupManager->get($groupid);
 				// Check if group exists
 				if ($group === null) {
-					throw new OCSException('Subadmin group does not exist',  102);
+					throw new OCSException('Subadmin group does not exist', 102);
 				}
 				// Check if trying to make subadmin of admin group
 				if ($group->getGID() === 'admin') {
@@ -1311,7 +1310,7 @@ public function addSubAdmin(string $userId, string $groupid): DataResponse {
 		}
 		// Check if group exists
 		if ($group === null) {
-			throw new OCSException('Group does not exist',  102);
+			throw new OCSException('Group does not exist', 102);
 		}
 		// Check if trying to make subadmin of admin group
 		if ($group->getGID() === 'admin') {

lib/private/Files/View.php

@@ -164,7 +164,7 @@ public function getRoot() {
 	 * get path relative to the root of the view
 	 *
 	 * @param string $path
-	 * @return string
+	 * @return ?string
 	 */
 	public function getRelativePath($path) {
 		$this->assertPathLength($path);
@@ -1241,7 +1241,7 @@ private function basicOperation($operation, $path, $hooks = [], $extraParam = nu
 	 * get the path relative to the default root for hook usage
 	 *
 	 * @param string $path
-	 * @return string
+	 * @return ?string
 	 */
 	private function getHookPath($path) {
 		if (!Filesystem::getView()) {

lib/private/User/User.php

@@ -529,6 +529,7 @@ public function setQuota($quota) {
 			$this->config->setUserValue($this->uid, 'files', 'quota', $quota);
 			$this->triggerChange('quota', $quota, $oldQuota);
 		}
+		\OC_Helper::clearStorageInfo('/' . $this->uid . '/files');
 	}
 
 	/**

lib/private/legacy/OC_Helper.php

@@ -473,7 +473,7 @@ public static function getStorageInfo($path, $rootInfo = null, $includeMountPoin
 		if (!$view) {
 			throw new \OCP\Files\NotFoundException();
 		}
-		$fullPath = $view->getAbsolutePath($path);
+		$fullPath = Filesystem::normalizePath($view->getAbsolutePath($path));
 
 		$cacheKey = $fullPath. '::' . ($includeMountPoints ? 'include' : 'exclude');
 		if ($useCache) {
@@ -620,6 +620,15 @@ private static function getGlobalStorageInfo(int|float $quota, IUser $user, IMou
 		];
 	}
 
+	public static function clearStorageInfo(string $absolutePath): void {
+		/** @var ICacheFactory $cacheFactory */
+		$cacheFactory = \OC::$server->get(ICacheFactory::class);
+		$memcache = $cacheFactory->createLocal('storage_info');
+		$cacheKeyPrefix = Filesystem::normalizePath($absolutePath) . '::';
+		$memcache->remove($cacheKeyPrefix . 'include');
+		$memcache->remove($cacheKeyPrefix . 'exclude');
+	}
+
 	/**
 	 * Returns whether the config file is set manually to read-only
 	 * @return bool