diff --git a/3rdparty b/3rdparty index 1145cdd2a4b99..e3123f6fa0555 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 1145cdd2a4b998183f143e48ff2834d7865ed4e5 +Subproject commit e3123f6fa0555a5e951d70f530097f8c94ad170c diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 4c4d42f8d1525..ba3098f577e34 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -201,6 +201,9 @@ public function put($data) { } } + $lengthHeader = $this->request->getHeader('content-length'); + $expected = $lengthHeader !== '' ? (int)$lengthHeader : null; + if ($partStorage->instanceOfStorage(IWriteStreamStorage::class)) { $isEOF = false; $wrappedData = CallbackWrapper::wrap($data, null, null, null, null, function ($stream) use (&$isEOF): void { @@ -212,7 +215,7 @@ public function put($data) { $count = -1; try { /** @var IWriteStreamStorage $partStorage */ - $count = $partStorage->writeStream($internalPartPath, $wrappedData); + $count = $partStorage->writeStream($internalPartPath, $wrappedData, $expected); } catch (GenericFileException $e) { $logger = Server::get(LoggerInterface::class); $logger->error('Error while writing stream to storage: ' . $e->getMessage(), ['exception' => $e, 'app' => 'webdav']); @@ -232,10 +235,7 @@ public function put($data) { [$count, $result] = \OC_Helper::streamCopy($data, $target); fclose($target); } - - $lengthHeader = $this->request->getHeader('content-length'); - $expected = $lengthHeader !== '' ? (int)$lengthHeader : -1; - if ($result === false && $expected >= 0) { + if ($result === false && $expected !== null) { throw new Exception( $this->l10n->t( 'Error while copying file to target location (copied: %1$s, expected filesize: %2$s)', @@ -250,7 +250,7 @@ public function put($data) { // if content length is sent by client: // double check if the file was fully received // compare expected and actual size - if ($expected >= 0 + if ($expected !== null && $expected !== $count && $this->request->getMethod() === 'PUT' ) { diff --git a/apps/files_external/3rdparty/composer.json b/apps/files_external/3rdparty/composer.json index 6af1429aa7fd7..3ae9569c3f242 100644 --- a/apps/files_external/3rdparty/composer.json +++ b/apps/files_external/3rdparty/composer.json @@ -9,6 +9,6 @@ }, "require": { "icewind/smb": "3.7.0", - "icewind/streams": "0.7.7" + "icewind/streams": "0.7.8" } } diff --git a/apps/files_external/3rdparty/composer.lock b/apps/files_external/3rdparty/composer.lock index b0cf919de9cba..7e44f13bcca43 100644 --- a/apps/files_external/3rdparty/composer.lock +++ b/apps/files_external/3rdparty/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b75bb2715ed2dae7d090ae40b9843a2", + "content-hash": "565bb9d1046efbd95b08a2b4df1a4efb", "packages": [ { "name": "icewind/smb", @@ -55,17 +55,11 @@ }, { "name": "icewind/streams", - "version": "v0.7.7", + "version": "v0.7.8", "source": { "type": "git", - "url": "https://github.com/icewind1991/Streams.git", - "reference": "64200fd7cfcc7f550c3c695c48d8fd8bba97fecb" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/icewind1991/Streams/zipball/64200fd7cfcc7f550c3c695c48d8fd8bba97fecb", - "reference": "64200fd7cfcc7f550c3c695c48d8fd8bba97fecb", - "shasum": "" + "url": "https://codeberg.org/icewind/streams", + "reference": "cb2bd3ed41b516efb97e06e8da35a12ef58ba48b" }, "require": { "php": ">=7.1" @@ -92,11 +86,7 @@ } ], "description": "A set of generic stream wrappers", - "support": { - "issues": "https://github.com/icewind1991/Streams/issues", - "source": "https://github.com/icewind1991/Streams/tree/v0.7.7" - }, - "time": "2023-03-16T14:52:25+00:00" + "time": "2024-12-05T14:36:22+00:00" } ], "packages-dev": [], diff --git a/apps/files_external/3rdparty/composer/InstalledVersions.php b/apps/files_external/3rdparty/composer/InstalledVersions.php index 51e734a774b3e..6d29bff66aace 100644 --- a/apps/files_external/3rdparty/composer/InstalledVersions.php +++ b/apps/files_external/3rdparty/composer/InstalledVersions.php @@ -32,6 +32,11 @@ class InstalledVersions */ private static $installed; + /** + * @var bool + */ + private static $installedIsLocalDir; + /** * @var bool|null */ @@ -309,6 +314,12 @@ public static function reload($data) { self::$installed = $data; self::$installedByVendor = array(); + + // when using reload, we disable the duplicate protection to ensure that self::$installed data is + // always returned, but we cannot know whether it comes from the installed.php in __DIR__ or not, + // so we have to assume it does not, and that may result in duplicate data being returned when listing + // all installed packages for example + self::$installedIsLocalDir = false; } /** @@ -322,19 +333,27 @@ private static function getInstalled() } $installed = array(); + $copiedLocalDir = false; if (self::$canGetVendors) { + $selfDir = strtr(__DIR__, '\\', '/'); foreach (ClassLoader::getRegisteredLoaders() as $vendorDir => $loader) { + $vendorDir = strtr($vendorDir, '\\', '/'); if (isset(self::$installedByVendor[$vendorDir])) { $installed[] = self::$installedByVendor[$vendorDir]; } elseif (is_file($vendorDir.'/composer/installed.php')) { /** @var array{root: array{name: string, pretty_version: string, version: string, reference: string|null, type: string, install_path: string, aliases: string[], dev: bool}, versions: array} $required */ $required = require $vendorDir.'/composer/installed.php'; - $installed[] = self::$installedByVendor[$vendorDir] = $required; - if (null === self::$installed && strtr($vendorDir.'/composer', '\\', '/') === strtr(__DIR__, '\\', '/')) { - self::$installed = $installed[count($installed) - 1]; + self::$installedByVendor[$vendorDir] = $required; + $installed[] = $required; + if (self::$installed === null && $vendorDir.'/composer' === $selfDir) { + self::$installed = $required; + self::$installedIsLocalDir = true; } } + if (self::$installedIsLocalDir && $vendorDir.'/composer' === $selfDir) { + $copiedLocalDir = true; + } } } @@ -350,7 +369,7 @@ private static function getInstalled() } } - if (self::$installed !== array()) { + if (self::$installed !== array() && !$copiedLocalDir) { $installed[] = self::$installed; } diff --git a/apps/files_external/3rdparty/composer/installed.json b/apps/files_external/3rdparty/composer/installed.json index 275a586f4fb44..9d24d06a77718 100644 --- a/apps/files_external/3rdparty/composer/installed.json +++ b/apps/files_external/3rdparty/composer/installed.json @@ -52,18 +52,12 @@ }, { "name": "icewind/streams", - "version": "v0.7.7", - "version_normalized": "0.7.7.0", + "version": "v0.7.8", + "version_normalized": "0.7.8.0", "source": { "type": "git", - "url": "https://github.com/icewind1991/Streams.git", - "reference": "64200fd7cfcc7f550c3c695c48d8fd8bba97fecb" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/icewind1991/Streams/zipball/64200fd7cfcc7f550c3c695c48d8fd8bba97fecb", - "reference": "64200fd7cfcc7f550c3c695c48d8fd8bba97fecb", - "shasum": "" + "url": "https://codeberg.org/icewind/streams", + "reference": "cb2bd3ed41b516efb97e06e8da35a12ef58ba48b" }, "require": { "php": ">=7.1" @@ -73,9 +67,9 @@ "phpstan/phpstan": "^0.12", "phpunit/phpunit": "^9" }, - "time": "2023-03-16T14:52:25+00:00", + "time": "2024-12-05T14:36:22+00:00", "type": "library", - "installation-source": "dist", + "installation-source": "source", "autoload": { "psr-4": { "Icewind\\Streams\\": "src/" @@ -92,10 +86,6 @@ } ], "description": "A set of generic stream wrappers", - "support": { - "issues": "https://github.com/icewind1991/Streams/issues", - "source": "https://github.com/icewind1991/Streams/tree/v0.7.7" - }, "install-path": "../icewind/streams" } ], diff --git a/apps/files_external/3rdparty/composer/installed.php b/apps/files_external/3rdparty/composer/installed.php index 12058ec947445..81b98dacabaec 100644 --- a/apps/files_external/3rdparty/composer/installed.php +++ b/apps/files_external/3rdparty/composer/installed.php @@ -3,7 +3,7 @@ 'name' => 'files_external/3rdparty', 'pretty_version' => 'dev-master', 'version' => 'dev-master', - 'reference' => '101c7575ae7684a572e53740c63635cd12685995', + 'reference' => '05e64418a77134e55824fc8c1a91125805428f3f', 'type' => 'library', 'install_path' => __DIR__ . '/../', 'aliases' => array(), @@ -13,7 +13,7 @@ 'files_external/3rdparty' => array( 'pretty_version' => 'dev-master', 'version' => 'dev-master', - 'reference' => '101c7575ae7684a572e53740c63635cd12685995', + 'reference' => '05e64418a77134e55824fc8c1a91125805428f3f', 'type' => 'library', 'install_path' => __DIR__ . '/../', 'aliases' => array(), @@ -29,9 +29,9 @@ 'dev_requirement' => false, ), 'icewind/streams' => array( - 'pretty_version' => 'v0.7.7', - 'version' => '0.7.7.0', - 'reference' => '64200fd7cfcc7f550c3c695c48d8fd8bba97fecb', + 'pretty_version' => 'v0.7.8', + 'version' => '0.7.8.0', + 'reference' => 'cb2bd3ed41b516efb97e06e8da35a12ef58ba48b', 'type' => 'library', 'install_path' => __DIR__ . '/../icewind/streams', 'aliases' => array(), diff --git a/apps/files_external/3rdparty/icewind/streams/src/CountWrapper.php b/apps/files_external/3rdparty/icewind/streams/src/CountWrapper.php index d1427dd97afa6..eb2ba9a92969d 100644 --- a/apps/files_external/3rdparty/icewind/streams/src/CountWrapper.php +++ b/apps/files_external/3rdparty/icewind/streams/src/CountWrapper.php @@ -60,6 +60,17 @@ protected function open() { return true; } + public function stream_seek($offset, $whence = SEEK_SET) { + if ($whence === SEEK_SET) { + $this->readCount = $offset; + $this->writeCount = $offset; + } else if ($whence === SEEK_CUR) { + $this->readCount += $offset; + $this->writeCount += $offset; + } + return parent::stream_seek($offset, $whence); + } + public function dir_opendir($path, $options) { return $this->open(); } diff --git a/lib/private/Files/ObjectStore/IObjectStoreMetaData.php b/lib/private/Files/ObjectStore/IObjectStoreMetaData.php index dc6b55076acb9..945c1133b0bbb 100644 --- a/lib/private/Files/ObjectStore/IObjectStoreMetaData.php +++ b/lib/private/Files/ObjectStore/IObjectStoreMetaData.php @@ -33,4 +33,13 @@ public function getObjectMetaData(string $urn): array; * @since 32.0.0 */ public function listObjects(string $prefix = ''): \Iterator; + + /** + * @param string $urn the unified resource name used to identify the object + * @param resource $stream stream with the data to write + * @param ObjectMetaData $metaData the metadata to set for the object + * @throws \Exception when something goes wrong, message will be logged + * @since 32.0.0 + */ + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void; } diff --git a/lib/private/Files/ObjectStore/ObjectStoreStorage.php b/lib/private/Files/ObjectStore/ObjectStoreStorage.php index cd09f8705b030..60fb618aa2d2a 100644 --- a/lib/private/Files/ObjectStore/ObjectStoreStorage.php +++ b/lib/private/Files/ObjectStore/ObjectStoreStorage.php @@ -479,6 +479,12 @@ public function writeStream(string $path, $stream, ?int $size = null): int { $mimetypeDetector = \OC::$server->getMimeTypeDetector(); $mimetype = $mimetypeDetector->detectPath($path); + $metadata = [ + 'mimetype' => $mimetype, + ]; + if ($size) { + $metadata['size'] = $size; + } $stat['mimetype'] = $mimetype; $stat['etag'] = $this->getETag($path); @@ -500,24 +506,27 @@ public function writeStream(string $path, $stream, ?int $size = null): int { $urn = $this->getURN($fileId); try { //upload to object storage - if ($size === null) { - $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) { + + $totalWritten = 0; + $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) { + if (is_null($size) && !$exists) { $this->getCache()->update($fileId, [ 'size' => $writtenSize, ]); - $size = $writtenSize; - }); - $this->objectStore->writeObject($urn, $countStream, $mimetype); - if (is_resource($countStream)) { - fclose($countStream); } - $stat['size'] = $size; + $totalWritten = $writtenSize; + }); + + if ($this->objectStore instanceof IObjectStoreMetaData) { + $this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata); } else { - $this->objectStore->writeObject($urn, $stream, $mimetype); - if (is_resource($stream)) { - fclose($stream); - } + $this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']); + } + if (is_resource($countStream)) { + fclose($countStream); } + + $stat['size'] = $totalWritten; } catch (\Exception $ex) { if (!$exists) { /* @@ -541,7 +550,7 @@ public function writeStream(string $path, $stream, ?int $size = null): int { ] ); } - throw $ex; // make this bubble up + throw new GenericFileException('Error while writing stream to object store', 0, $ex); } if ($exists) { @@ -557,7 +566,7 @@ public function writeStream(string $path, $stream, ?int $size = null): int { } } - return $size; + return $totalWritten; } public function getObjectStore(): IObjectStore { diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 93c92a47a714b..7f4fa13ad5de0 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -6,6 +6,8 @@ */ namespace OC\Files\ObjectStore; +use Aws\Command; +use Aws\Exception\MultipartUploadException; use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\MultipartCopy; use Aws\S3\MultipartUploader; @@ -83,18 +85,24 @@ public function readObject($urn) { * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeSingle(string $urn, StreamInterface $stream, ?string $mimetype = null): void { - $this->getConnection()->putObject([ + protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void { + $args = [ 'Bucket' => $this->bucket, 'Key' => $urn, 'Body' => $stream, 'ACL' => 'private', - 'ContentType' => $mimetype, + 'ContentType' => $metaData['mimetype'] ?? null, 'StorageClass' => $this->storageClass, - ] + $this->getSSECParameters()); + ] + $this->getSSECParameters(); + + if ($size = $stream->getSize()) { + $args['ContentLength'] = $size; + } + + $this->getConnection()->putObject($args); } @@ -103,15 +111,17 @@ protected function writeSingle(string $urn, StreamInterface $stream, ?string $mi * * @param string $urn the unified resource name used to identify the object * @param StreamInterface $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object + * @param array $metaData the metadata to set for the object * @throws \Exception when something goes wrong, message will be logged */ - protected function writeMultiPart(string $urn, StreamInterface $stream, ?string $mimetype = null): void { + protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void { $attempts = 0; $uploaded = false; $concurrency = $this->concurrency; $exception = null; $state = null; + $size = $stream->getSize(); + $totalWritten = 0; // retry multipart upload once with concurrency at half on failure while (!$uploaded && $attempts <= 1) { @@ -122,9 +132,18 @@ protected function writeMultiPart(string $urn, StreamInterface $stream, ?string 'part_size' => $this->uploadPartSize, 'state' => $state, 'params' => [ - 'ContentType' => $mimetype, + 'ContentType' => $metaData['mimetype'] ?? null, 'StorageClass' => $this->storageClass, ] + $this->getSSECParameters(), + 'before_upload' => function (Command $command) use (&$totalWritten) { + $totalWritten += $command['ContentLength']; + }, + 'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) { + if ($size !== null && $totalWritten != $size) { + $e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten); + throw new MultipartUploadException($uploader->getState(), $e); + } + }, ]); try { @@ -141,6 +160,9 @@ protected function writeMultiPart(string $urn, StreamInterface $stream, ?string if ($stream->isSeekable()) { $stream->rewind(); } + } catch (MultipartUploadException $e) { + $exception = $e; + break; } } @@ -156,17 +178,19 @@ protected function writeMultiPart(string $urn, StreamInterface $stream, ?string } } - - /** - * @param string $urn the unified resource name used to identify the object - * @param resource $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 - * @throws \Exception when something goes wrong, message will be logged - * @since 7.0.0 - */ public function writeObject($urn, $stream, ?string $mimetype = null) { + $metaData = []; + if ($mimetype) { + $metaData['mimetype'] = $mimetype; + } + $this->writeObjectWithMetaData($urn, $stream, $metaData); + } + + public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void { $canSeek = fseek($stream, 0, SEEK_CUR) === 0; - $psrStream = Utils::streamFor($stream); + $psrStream = Utils::streamFor($stream, [ + 'size' => $metaData['size'] ?? null, + ]); $size = $psrStream->getSize(); @@ -179,16 +203,16 @@ public function writeObject($urn, $stream, ?string $mimetype = null) { $buffer->seek(0); if ($buffer->getSize() < $this->putSizeLimit) { // buffer is fully seekable, so use it directly for the small upload - $this->writeSingle($urn, $buffer, $mimetype); + $this->writeSingle($urn, $buffer, $metaData); } else { $loadStream = new Psr7\AppendStream([$buffer, $psrStream]); - $this->writeMultiPart($urn, $loadStream, $mimetype); + $this->writeMultiPart($urn, $loadStream, $metaData); } } else { if ($size < $this->putSizeLimit) { - $this->writeSingle($urn, $psrStream, $mimetype); + $this->writeSingle($urn, $psrStream, $metaData); } else { - $this->writeMultiPart($urn, $psrStream, $mimetype); + $this->writeMultiPart($urn, $psrStream, $metaData); } } $psrStream->close();