diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index d88ef0ac8e719..dcbb5f84ca611 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -1,7 +1,6 @@ - * * @author Arthur Schiwon * @author Christoph Wurst * @author Florent @@ -10,7 +9,6 @@ * @author Roeland Jago Douma * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com> * @author Stephen Cuppett - * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -25,15 +23,14 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - * */ namespace OC\Files\ObjectStore; use Aws\ClientResolver; use Aws\Credentials\CredentialProvider; -use Aws\Credentials\EcsCredentialProvider; use Aws\Credentials\Credentials; +use Aws\Credentials\EcsCredentialProvider; use Aws\Exception\CredentialsException; use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; @@ -60,25 +57,39 @@ trait S3ConnectionTrait { /** @var int */ protected $uploadPartSize; + /** @var string */ + protected $sseKmsKeyId; + + /** @var string */ + protected $sseKmsBucketKeyId; + protected $test; protected function parseParams($params) { if (empty($params['bucket'])) { - throw new \Exception("Bucket has to be configured."); + throw new \Exception('Bucket has to be configured.'); } - $this->id = 'amazon::' . $params['bucket']; + $this->id = 'amazon::'.$params['bucket']; $this->test = isset($params['test']); $this->bucket = $params['bucket']; $this->timeout = !isset($params['timeout']) ? 15 : $params['timeout']; $this->uploadPartSize = !isset($params['uploadPartSize']) ? 524288000 : $params['uploadPartSize']; $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; - $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; + $params['hostname'] = empty($params['hostname']) ? 's3.'.$params['region'].'.amazonaws.com' : $params['hostname']; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } - $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; + $params['autocreate'] = !isset($params['autocreate']) ? false : $params['autocreate']; + + // this avoid at least the hash lookups for each read/weite operation + if (isset($params['ssekmsbucketkeyid'])) { + $this->sseKmsBucketKeyId = $params['ssekmsbucketkeyid']; + } elseif (isset($params['ssekmskeyid'])) { + $this->sseKmsKeyId = $params['ssekmskeyid']; + } + $this->params = $params; } @@ -87,9 +98,129 @@ public function getBucket() { } /** - * Returns the connection + * Add the SSE KMS parameterdepending on the + * KMS encryption strategy (bucket, individual or + * no encryption) for object creations. + * + * @return array with encryption parameters + */ + public function getSseKmsPutParameters(): array { + if (!empty($this->sseKmsBucketKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + ]; + } elseif (!empty($this->sseKmsKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsKeyId, + ]; + } else { + return []; + } + } + + /** + * Add the SSE KMS parameter depending on the + * KMS encryption strategy (bucket, individual or + * no encryption) for object read. + * + * @return array with encryption parameters + */ + public function getSseKmsGetParameters(): array { + if (!empty($this->sseKmsBucketKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsBucketKeyId, + ]; + } elseif (!empty($this->sseKmsKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsKeyId, + ]; + } else { + return []; + } + } + + + /** + * Create the required bucket + * + * @throws \Exception if bucket creation fails + */ + protected function createNewBucket() { + $logger = \OC::$server->getLogger(); + try { + $logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket); + } + $this->connection->createBucket(['Bucket' => $this->bucket]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->logException($e, [ + 'message' => 'Invalid remote storage.', + 'level' => ILogger::DEBUG, + 'app' => 'objectstore', + ]); + throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage()); + } + } + + /** + * Check bucket key consistency or put bucket key if missing + * This operation only works for bucket owner or with + * s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission + * + * We recommend to use autocreate only on initial setup and + * use an S3:user only with object operation permission and no bucket operation permissions + * later with autocreate=false + * + * @throws \Exception if bucket key config is inconsistent or if putting the key fails + */ + protected function checkOrPutBucketKey() { + $logger = \OC::$server->getLogger(); + + try { + $encrypt_state = $this->connection->getBucketEncryption([ + 'Bucket' => $this->bucket, + ]); + return; + } catch (S3Exception $e) { + try { + $logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']); + $this->connection->putBucketEncryption([ + 'Bucket' => $this->bucket , + 'ServerSideEncryptionConfiguration' => [ + 'Rules' => [ + [ + 'ApplyServerSideEncryptionByDefault' => [ + 'KMSMasterKeyID' => $this->sseKmsBucketKeyId, + 'SSEAlgorithm' => 'aws:kms', + ], + 'BucketKeyEnabled' => true, + ], + ], + ], + ]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->logException($e, [ + 'message' => 'Bucket key problem.', + 'level' => ILogger::DEBUG, + 'app' => 'objectstore', + ]); + throw new \Exception('Putting configured bucket key to "'.$this->bucket.'" failed. '.$e->getMessage()); + } + } + } + + + /** + * Returns the connection. * * @return S3Client connected client + * * @throws \Exception if connection could not be made */ public function getConnection() { @@ -98,7 +229,7 @@ public function getConnection() { } $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; - $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; + $base_url = $scheme.'://'.$this->params['hostname'].':'.$this->params['port'].'/'; // Adding explicit credential provider to the beginning chain. // Including environment variables and IAM instance profiles. @@ -132,27 +263,16 @@ public function getConnection() { if (!$this->connection::isBucketDnsCompatible($this->bucket)) { $logger = \OC::$server->getLogger(); - $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', - ['app' => 'objectstore']); + $logger->debug('Bucket "'.$this->bucket.'" This bucket name is not dns compatible, it may contain invalid characters.', + ['app' => 'objectstore']); } - if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { - $logger = \OC::$server->getLogger(); - try { - $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); - } - $this->connection->createBucket(['Bucket' => $this->bucket]); - $this->testTimeout(); - } catch (S3Exception $e) { - $logger->logException($e, [ - 'message' => 'Invalid remote storage.', - 'level' => ILogger::DEBUG, - 'app' => 'objectstore', - ]); - throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); - } + if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) { + $this->createNewBucket(); + } + + if ($this->params['autocreate'] && isset($this->params['ssekmsbucketkeyid'])) { + $this->checkOrPutBucketKey(); } // google cloud's s3 compatibility doesn't like the EncodingType parameter @@ -164,7 +284,7 @@ public function getConnection() { } /** - * when running the tests wait to let the buckets catch up + * when running the tests wait to let the buckets catch up. */ private function testTimeout() { if ($this->test) { @@ -183,9 +303,9 @@ public static function legacySignatureProvider($version, $service, $region) { } /** - * This function creates a credential provider based on user parameter file + * This function creates a credential provider based on user parameter file. */ - protected function paramCredentialProvider() : callable { + protected function paramCredentialProvider(): callable { return function () { $key = empty($this->params['key']) ? null : $this->params['key']; $secret = empty($this->params['secret']) ? null : $this->params['secret']; @@ -197,6 +317,7 @@ protected function paramCredentialProvider() : callable { } $msg = 'Could not find parameters set for credentials in config file.'; + return new RejectedPromise(new CredentialsException($msg)); }; } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 250f8fd1edd01..97a588ca2e141 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -1,13 +1,11 @@ - * * @author Christoph Wurst * @author Florent * @author Morris Jobke * @author Robin Appelman * @author Roeland Jago Douma - * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -22,7 +20,6 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - * */ namespace OC\Files\ObjectStore; @@ -36,26 +33,37 @@ trait S3ObjectTrait { /** - * Returns the connection + * Returns the connection. * * @return S3Client connected client + * * @throws \Exception if connection could not be made */ abstract protected function getConnection(); + /* compute configured encryption headers for put operations */ + abstract protected function getSseKmsPutParameters(); + + /* compute configured encryption headers for get operations */ + abstract protected function getSseKmsGetParameters(); + /** * @param string $urn the unified resource name used to identify the object + * * @return resource stream with the read data + * * @throws \Exception when something goes wrong, message will be logged + * * @since 7.0.0 */ public function readObject($urn) { return SeekableHttpStream::open(function ($range) use ($urn) { - $command = $this->getConnection()->getCommand('GetObject', [ + $s3params = [ 'Bucket' => $this->bucket, 'Key' => $urn, - 'Range' => 'bytes=' . $range, - ]); + 'Range' => 'bytes='.$range, + ] + $this->getSseKmsGetParameters(); + $command = $this->getConnection()->getCommand('GetObject', $s3params); $request = \Aws\serialize($command); $headers = []; foreach ($request->getHeaders() as $key => $values) { @@ -71,6 +79,7 @@ public function readObject($urn) { ]; $context = stream_context_create($opts); + return fopen($request->getUri(), 'r', false, $context); }); } @@ -88,32 +97,46 @@ public function writeObject($urn, $stream, string $mimetype = null) { $count += $read; }); - $uploader = new MultipartUploader($this->getConnection(), $countStream, [ + $s3params = [ 'bucket' => $this->bucket, 'key' => $urn, 'part_size' => $this->uploadPartSize, 'params' => [ 'ContentType' => $mimetype - ] - ]); + ] + $this->getSseKmsPutParameters(), + ]; + $uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params); try { $uploader->upload(); } catch (S3MultipartUploadException $e) { // This is an empty file so just touch it then if ($count === 0 && feof($countStream)) { - $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, ''); + $s3params = [ + 'params' => $this->getSseKmsPutParameters(), + ]; + $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params); $uploader->upload(); } else { throw $e; } + } finally { + // this handles [S3] fclose(): supplied resource is not a valid stream resource #23373 + // see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555 + // which also recommends the solution + if (is_resource($countStream)) { + fclose($countStream); + } } } /** * @param string $urn the unified resource name used to identify the object + * * @return void + * * @throws \Exception when something goes wrong, message will be logged + * * @since 7.0.0 */ public function deleteObject($urn) { @@ -126,8 +149,13 @@ public function deleteObject($urn) { public function objectExists($urn) { return $this->getConnection()->doesObjectExist($this->bucket, $urn); } - + + /** + * S3 copy command with SSE KMS key handling. + */ public function copyObject($from, $to) { - $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to); + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ + 'params' => $this->getSseKmsPutParameters(), + ]); } } diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index 9781421238277..ebffdcdc74e63 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -62,7 +62,7 @@ public function stream_seek($offset, $whence = SEEK_SET) { class S3Test extends ObjectStoreTest { protected function getInstance() { $config = \OC::$server->getConfig()->getSystemValue('objectstore'); - if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') { + if (!is_array($config) || $config['class'] !== '\\OC\\Files\\ObjectStore\\S3') { $this->markTestSkipped('objectstore not configured for s3'); } @@ -70,11 +70,6 @@ protected function getInstance() { } public function testUploadNonSeekable() { - $config = \OC::$server->getConfig()->getSystemValue('objectstore'); - if (!is_array($config) || $config['class'] !== 'OC\\Files\\ObjectStore\\S3') { - $this->markTestSkipped('objectstore not configured for s3'); - } - $s3 = $this->getInstance(); $s3->writeObject('multiparttest', NonSeekableStream::wrap(fopen(__FILE__, 'r')));