From 61d66d79544e3bd17a4f96a569fc24596ec414db Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 14 Feb 2019 15:51:40 +0100 Subject: [PATCH 1/4] Fix empty file uploads to S3 Signed-off-by: Roeland Jago Douma --- .../Files/ObjectStore/S3ObjectTrait.php | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 0b55c319ea8a4..7c46ba25c1126 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -23,8 +23,11 @@ namespace OC\Files\ObjectStore; +use Aws\S3\Exception\S3MultipartUploadException; use Aws\S3\MultipartUploader; +use Aws\S3\ObjectUploader; use Aws\S3\S3Client; +use Icewind\Streams\CallbackWrapper; const S3_UPLOAD_PART_SIZE = 524288000; // 500MB @@ -73,12 +76,30 @@ function readObject($urn) { * @since 7.0.0 */ function writeObject($urn, $stream) { - $uploader = new MultipartUploader($this->getConnection(), $stream, [ + $count = 0; + $countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) { + $count += $read; + }); + + $uploader = new MultipartUploader($this->getConnection(), $countStream, [ 'bucket' => $this->bucket, 'key' => $urn, 'part_size' => S3_UPLOAD_PART_SIZE ]); - $uploader->upload(); + + try { + $uploader->upload(); + } catch (S3MultipartUploadException $e) { + // This is an emty file so just touch it then + if ($count === 0 && feof($countStream)) { + $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, ''); + $uploader->upload(); + } else { + throw $e; + } + } + + fclose($countStream); } /** From 3720add736e7a3974ff869d92bca5e454ec302a4 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 14 Feb 2019 16:46:30 +0100 Subject: [PATCH 2/4] Add a wrapper to determine if a file is EOF The stream is already closed at this point. Which means feof will always return false. We have to determine if the stream is EOF in the preCloseCallback. And pass this info along. Then the logic works as expected. Signed-off-by: Roeland Jago Douma --- apps/dav/lib/Connector/Sabre/File.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index f948f0f552d31..7767e83f50849 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -36,6 +36,7 @@ namespace OCA\DAV\Connector\Sabre; +use Icewind\Streams\CallbackWrapper; use OC\AppFramework\Http\Request; use OC\Files\Filesystem; use OC\Files\View; @@ -166,10 +167,15 @@ public function put($data) { } if ($partStorage->instanceOfStorage(Storage\IWriteStreamStorage::class)) { - $count = $partStorage->writeStream($internalPartPath, $data); + $isEOF = false; + $wrappedData = CallbackWrapper::wrap($data, null, null, null, null, function($stream) use (&$isEOF) { + $isEOF = feof($stream); + }); + + $count = $partStorage->writeStream($internalPartPath, $wrappedData); $result = $count > 0; if ($result === false) { - $result = feof($data); + $result = $isEOF; } } else { From 3edffdd5818439f6b4710e50ba32972c875091ea Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 15 Feb 2019 08:41:52 +0100 Subject: [PATCH 3/4] Bump icewind/streams for supported streams Signed-off-by: Roeland Jago Douma --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index 42b675142d5c3..7c64d231f833b 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 42b675142d5c33b585f1bb204e910afeed7e98db +Subproject commit 7c64d231f833b8ab8f9dcd017499c8c18b7b78f9 From fc967a5ac2dee93376685d4fe9faababafe55491 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 18 Feb 2019 14:25:57 +0100 Subject: [PATCH 4/4] Make sure that we have a stream Signed-off-by: Roeland Jago Douma --- apps/dav/lib/Connector/Sabre/File.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 7767e83f50849..388bcff92065d 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -167,6 +167,13 @@ public function put($data) { } if ($partStorage->instanceOfStorage(Storage\IWriteStreamStorage::class)) { + + if (!is_resource($data)) { + $data = fopen('php://temp', 'r+'); + fwrite($data, 'foobar'); + rewind($data); + } + $isEOF = false; $wrappedData = CallbackWrapper::wrap($data, null, null, null, null, function($stream) use (&$isEOF) { $isEOF = feof($stream);