Skip to content

Commit 27486cf

Browse files
authored
Merge pull request #20966 from nextcloud/backport/20033/stable17
[stable17] Enable fseek for files in S3 storage
2 parents 76e3d63 + e471c37 commit 27486cf

File tree

6 files changed

+236
-23
lines changed

6 files changed

+236
-23
lines changed

lib/composer/composer/autoload_classmap.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -930,6 +930,7 @@
930930
'OC\\Files\\Storage\\Wrapper\\Wrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/Wrapper.php',
931931
'OC\\Files\\Stream\\Encryption' => $baseDir . '/lib/private/Files/Stream/Encryption.php',
932932
'OC\\Files\\Stream\\Quota' => $baseDir . '/lib/private/Files/Stream/Quota.php',
933+
'OC\\Files\\Stream\\SeekableHttpStream' => $baseDir . '/lib/private/Files/Stream/SeekableHttpStream.php',
933934
'OC\\Files\\Type\\Detection' => $baseDir . '/lib/private/Files/Type/Detection.php',
934935
'OC\\Files\\Type\\Loader' => $baseDir . '/lib/private/Files/Type/Loader.php',
935936
'OC\\Files\\Type\\TemplateManager' => $baseDir . '/lib/private/Files/Type/TemplateManager.php',

lib/composer/composer/autoload_static.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
964964
'OC\\Files\\Storage\\Wrapper\\Wrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Wrapper.php',
965965
'OC\\Files\\Stream\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Encryption.php',
966966
'OC\\Files\\Stream\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Quota.php',
967+
'OC\\Files\\Stream\\SeekableHttpStream' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/SeekableHttpStream.php',
967968
'OC\\Files\\Type\\Detection' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Detection.php',
968969
'OC\\Files\\Type\\Loader' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Loader.php',
969970
'OC\\Files\\Type\\TemplateManager' => __DIR__ . '/../../..' . '/lib/private/Files/Type/TemplateManager.php',

lib/private/Files/ObjectStore/S3ObjectTrait.php

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
use Aws\S3\ObjectUploader;
2929
use Aws\S3\S3Client;
3030
use Icewind\Streams\CallbackWrapper;
31+
use OC\Files\Stream\SeekableHttpStream;
3132

3233
const S3_UPLOAD_PART_SIZE = 524288000; // 500MB
3334

@@ -47,27 +48,29 @@ abstract protected function getConnection();
4748
* @since 7.0.0
4849
*/
4950
function readObject($urn) {
50-
$client = $this->getConnection();
51-
$command = $client->getCommand('GetObject', [
52-
'Bucket' => $this->bucket,
53-
'Key' => $urn
54-
]);
55-
$request = \Aws\serialize($command);
56-
$headers = [];
57-
foreach ($request->getHeaders() as $key => $values) {
58-
foreach ($values as $value) {
59-
$headers[] = "$key: $value";
51+
return SeekableHttpStream::open(function ($range) use ($urn) {
52+
$command = $this->getConnection()->getCommand('GetObject', [
53+
'Bucket' => $this->bucket,
54+
'Key' => $urn,
55+
'Range' => 'bytes=' . $range,
56+
]);
57+
$request = \Aws\serialize($command);
58+
$headers = [];
59+
foreach ($request->getHeaders() as $key => $values) {
60+
foreach ($values as $value) {
61+
$headers[] = "$key: $value";
62+
}
6063
}
61-
}
62-
$opts = [
63-
'http' => [
64-
'protocol_version' => 1.1,
65-
'header' => $headers
66-
]
67-
];
64+
$opts = [
65+
'http' => [
66+
'protocol_version' => 1.1,
67+
'header' => $headers,
68+
],
69+
];
6870

69-
$context = stream_context_create($opts);
70-
return fopen($request->getUri(), 'r', false, $context);
71+
$context = stream_context_create($opts);
72+
return fopen($request->getUri(), 'r', false, $context);
73+
});
7174
}
7275

7376
/**
@@ -85,7 +88,7 @@ function writeObject($urn, $stream) {
8588
$uploader = new MultipartUploader($this->getConnection(), $countStream, [
8689
'bucket' => $this->bucket,
8790
'key' => $urn,
88-
'part_size' => S3_UPLOAD_PART_SIZE
91+
'part_size' => S3_UPLOAD_PART_SIZE,
8992
]);
9093

9194
try {
@@ -112,7 +115,7 @@ function writeObject($urn, $stream) {
112115
function deleteObject($urn) {
113116
$this->getConnection()->deleteObject([
114117
'Bucket' => $this->bucket,
115-
'Key' => $urn
118+
'Key' => $urn,
116119
]);
117120
}
118121

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
<?php
2+
/**
3+
*
4+
* @copyright Copyright (c) 2020, Lukas Stabe ([email protected])
5+
*
6+
* @license GNU AGPL version 3 or any later version
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the GNU Affero General Public License as
10+
* published by the Free Software Foundation, either version 3 of the
11+
* License, or (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU Affero General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU Affero General Public License
19+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
20+
*
21+
*/
22+
23+
namespace OC\Files\Stream;
24+
25+
use Icewind\Streams\File;
26+
27+
/**
28+
* A stream wrapper that uses http range requests to provide a seekable stream for http reading
29+
*/
30+
class SeekableHttpStream implements File {
31+
private const PROTOCOL = 'httpseek';
32+
33+
private static $registered = false;
34+
35+
/**
36+
* Registers the stream wrapper using the `httpseek://` url scheme
37+
* $return void
38+
*/
39+
private static function registerIfNeeded() {
40+
if (!self::$registered) {
41+
stream_wrapper_register(
42+
self::PROTOCOL,
43+
self::class
44+
);
45+
self::$registered = true;
46+
}
47+
}
48+
49+
/**
50+
* Open a readonly-seekable http stream
51+
*
52+
* The provided callback will be called with byte range and should return an http stream for the requested range
53+
*
54+
* @param callable $callback
55+
* @return false|resource
56+
*/
57+
public static function open(callable $callback) {
58+
$context = stream_context_create([
59+
SeekableHttpStream::PROTOCOL => [
60+
'callback' => $callback
61+
],
62+
]);
63+
64+
SeekableHttpStream::registerIfNeeded();
65+
return fopen(SeekableHttpStream::PROTOCOL . '://', 'r', false, $context);
66+
}
67+
68+
/** @var resource */
69+
public $context;
70+
71+
/** @var callable */
72+
private $openCallback;
73+
74+
/** @var resource */
75+
private $current;
76+
/** @var int */
77+
private $offset = 0;
78+
79+
private function reconnect(int $start) {
80+
$range = $start . '-';
81+
if ($this->current != null) {
82+
fclose($this->current);
83+
}
84+
85+
$this->current = ($this->openCallback)($range);
86+
87+
if ($this->current === false) {
88+
return false;
89+
}
90+
91+
$responseHead = stream_get_meta_data($this->current)['wrapper_data'];
92+
$rangeHeaders = array_values(array_filter($responseHead, function ($v) {
93+
return preg_match('#^content-range:#i', $v) === 1;
94+
}));
95+
if (!$rangeHeaders) {
96+
return false;
97+
}
98+
$contentRange = $rangeHeaders[0];
99+
100+
$content = trim(explode(':', $contentRange)[1]);
101+
$range = trim(explode(' ', $content)[1]);
102+
$begin = intval(explode('-', $range)[0]);
103+
104+
if ($begin !== $start) {
105+
return false;
106+
}
107+
108+
$this->offset = $begin;
109+
110+
return true;
111+
}
112+
113+
public function stream_open($path, $mode, $options, &$opened_path) {
114+
$options = stream_context_get_options($this->context)[self::PROTOCOL];
115+
$this->openCallback = $options['callback'];
116+
117+
return $this->reconnect(0);
118+
}
119+
120+
public function stream_read($count) {
121+
if (!$this->current) {
122+
return false;
123+
}
124+
$ret = fread($this->current, $count);
125+
$this->offset += strlen($ret);
126+
return $ret;
127+
}
128+
129+
public function stream_seek($offset, $whence = SEEK_SET) {
130+
switch ($whence) {
131+
case SEEK_SET:
132+
if ($offset === $this->offset) {
133+
return true;
134+
}
135+
return $this->reconnect($offset);
136+
case SEEK_CUR:
137+
if ($offset === 0) {
138+
return true;
139+
}
140+
return $this->reconnect($this->offset + $offset);
141+
case SEEK_END:
142+
return false;
143+
}
144+
return false;
145+
}
146+
147+
public function stream_tell() {
148+
return $this->offset;
149+
}
150+
151+
public function stream_stat() {
152+
if (is_resource($this->current)) {
153+
return fstat($this->current);
154+
} else {
155+
return false;
156+
}
157+
}
158+
159+
public function stream_eof() {
160+
if (is_resource($this->current)) {
161+
return feof($this->current);
162+
} else {
163+
return true;
164+
}
165+
}
166+
167+
public function stream_close() {
168+
if (is_resource($this->current)) {
169+
fclose($this->current);
170+
}
171+
}
172+
173+
public function stream_write($data) {
174+
return false;
175+
}
176+
177+
public function stream_set_option($option, $arg1, $arg2) {
178+
return false;
179+
}
180+
181+
public function stream_truncate($size) {
182+
return false;
183+
}
184+
185+
public function stream_lock($operation) {
186+
return false;
187+
}
188+
189+
public function stream_flush() {
190+
return; //noop because readonly stream
191+
}
192+
}

tests/lib/Files/ObjectStore/ObjectStoreTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ abstract class ObjectStoreTest extends TestCase {
3131
*/
3232
abstract protected function getInstance();
3333

34-
private function stringToStream($data) {
34+
protected function stringToStream($data) {
3535
$stream = fopen('php://temp', 'w+');
3636
fwrite($stream, $data);
3737
rewind($stream);

tests/lib/Files/ObjectStore/S3Test.php

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
class MultiPartUploadS3 extends S3 {
2828
function writeObject($urn, $stream) {
2929
$this->getConnection()->upload($this->bucket, $urn, $stream, 'private', [
30-
'mup_threshold' => 1
30+
'mup_threshold' => 1,
3131
]);
3232
}
3333
}
@@ -83,4 +83,20 @@ public function testUploadNonSeekable() {
8383

8484
$this->assertEquals(file_get_contents(__FILE__), stream_get_contents($result));
8585
}
86+
87+
public function testSeek() {
88+
$data = file_get_contents(__FILE__);
89+
90+
$instance = $this->getInstance();
91+
$instance->writeObject('seek', $this->stringToStream($data));
92+
93+
$read = $instance->readObject('seek');
94+
$this->assertEquals(substr($data, 0, 100), fread($read, 100));
95+
96+
fseek($read, 10);
97+
$this->assertEquals(substr($data, 10, 100), fread($read, 100));
98+
99+
fseek($read, 100, SEEK_CUR);
100+
$this->assertEquals(substr($data, 210, 100), fread($read, 100));
101+
}
86102
}

0 commit comments

Comments
 (0)