From 6bf8b2e0aae2475ba460ac109593a2c877a46018 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 19 May 2020 02:31:20 -0700 Subject: [PATCH 1/5] [Storage][Blob] Added support for Object Replication --- sdk/storage/azure-storage-blob/CHANGELOG.md | 3 + .../azure/storage/blob/_deserialize.py | 36 +++- .../azure/storage/blob/_models.py | 7 + .../test_ors.test_ors_destination.yaml | 180 ++++++++++++++++++ .../recordings/test_ors.test_ors_source.yaml | 168 ++++++++++++++++ .../azure-storage-blob/tests/test_ors.py | 104 ++++++++++ 6 files changed, 493 insertions(+), 5 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml create mode 100644 sdk/storage/azure-storage-blob/tests/test_ors.py diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 6c836af26566..2797e2f5726e 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -2,6 +2,9 @@ ## 12.3.2 (Unreleased) +**New features** + +- Added support for object replication properties for download response and get properties. ## 12.3.1 (2020-04-29) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py index 857806b23193..00e0c084b729 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py @@ -22,6 +22,7 @@ def deserialize_blob_properties(response, obj, headers): metadata = deserialize_metadata(response, obj, headers) blob_properties = BlobProperties( metadata=metadata, + object_replication_source_properties=deserialize_ors_policies(response), **headers ) if 'Content-Range' in headers: @@ -32,6 +33,29 @@ def deserialize_blob_properties(response, obj, headers): return blob_properties +def deserialize_ors_policies(response): + # For source blobs (blobs that have policy ids and rule ids applied to them), + # the header will be formatted as "x-ms-or-_: {Complete, Failed}". + # The value of this header is the status of the replication. + or_policy_status_headers = {key: val for key, val in response.headers.items() + if key.startswith('x-ms-or') and key != 'x-ms-or-policy-id'} + + parsed_result = {} + + for key, val in or_policy_status_headers.items(): + policy_and_rule_ids = key[len('x-ms-or-'):].split('_') + policy_id = policy_and_rule_ids[0] + rule_id = policy_and_rule_ids[1] + + # we saw this policy already + if parsed_result.get(policy_id) is None: + parsed_result[policy_id] = {rule_id: val} + else: + parsed_result.get(policy_id)[rule_id] = val + + return parsed_result + + def deserialize_blob_stream(response, obj, headers): blob_properties = deserialize_blob_properties(response, obj, headers) obj.properties = blob_properties @@ -49,10 +73,10 @@ def deserialize_container_properties(response, obj, headers): def get_page_ranges_result(ranges): # type: (PageList) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]] - page_range = [] # type: ignore - clear_range = [] # type: List + page_range = [] # type: ignore + clear_range = [] # type: List if ranges.page_range: - page_range = [{'start': b.start, 'end': b.end} for b in ranges.page_range] # type: ignore + page_range = [{'start': b.start, 'end': b.end} for b in ranges.page_range] # type: ignore if ranges.clear_range: clear_range = [{'start': b.start, 'end': b.end} for b in ranges.clear_range] return page_range, clear_range # type: ignore @@ -73,11 +97,13 @@ def service_properties_deserialize(generated): """Deserialize a ServiceProperties objects into a dict. """ return { - 'analytics_logging': BlobAnalyticsLogging._from_generated(generated.logging), # pylint: disable=protected-access + 'analytics_logging': BlobAnalyticsLogging._from_generated(generated.logging), + # pylint: disable=protected-access 'hour_metrics': Metrics._from_generated(generated.hour_metrics), # pylint: disable=protected-access 'minute_metrics': Metrics._from_generated(generated.minute_metrics), # pylint: disable=protected-access 'cors': [CorsRule._from_generated(cors) for cors in generated.cors], # pylint: disable=protected-access 'target_version': generated.default_service_version, # pylint: disable=protected-access - 'delete_retention_policy': RetentionPolicy._from_generated(generated.delete_retention_policy), # pylint: disable=protected-access + 'delete_retention_policy': RetentionPolicy._from_generated(generated.delete_retention_policy), + # pylint: disable=protected-access 'static_website': StaticWebsite._from_generated(generated.static_website), # pylint: disable=protected-access } diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py index 79e74f1d70a4..82775a341f01 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py @@ -481,6 +481,11 @@ class BlobProperties(DictMixin): container-level scope is configured to allow overrides. Otherwise an error will be raised. :ivar bool request_server_encrypted: Whether this blob is encrypted. + :ivar dict(str, dict(str, str)) object_replication_source_properties: + Only present for blobs that have policy ids and rule ids applied to them. + DictionaryInvalidRangeThe\ + \ range specified is invalid for the current size of the resource.\nRequestId:cd614dd5-601e-007f-05c0-2d1559000000\n\ + Time:2020-05-19T09:30:31.3239333Z" + headers: + content-length: + - '249' + content-range: + - bytes */0 + content-type: + - application/xml + date: + - Tue, 19 May 2020 09:30:31 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-error-code: + - InvalidRange + x-ms-version: + - '2019-12-12' + status: + code: 416 + message: The range specified is invalid for the current size of the resource. +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Tue, 19 May 2020 09:30:31 GMT + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test2/bla.txt + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-disposition: + - '' + content-length: + - '0' + content-md5: + - 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: + - application/octet-stream + date: + - Tue, 19 May 2020 09:30:31 GMT + etag: + - '"0x8D7FB118A463E24"' + last-modified: + - Mon, 18 May 2020 09:55:04 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-blob-type: + - BlockBlob + x-ms-copy-completion-time: + - Mon, 18 May 2020 09:55:04 GMT + x-ms-copy-id: + - 47d2f0e0-9739-42f5-ad74-8359dbb0c2ec + x-ms-copy-progress: + - 0/0 + x-ms-copy-source: + - https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt?versionid=2020-05-18T09:53:04.5502688Z&sv=2015-04-05&ss=b&srt=sco&sp=rwdlacup&se=2020-05-19T09%3A13%3A27.6586322Z&spr=https + x-ms-copy-status: + - success + x-ms-creation-time: + - Mon, 18 May 2020 09:55:04 GMT + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-or-policy-id: + - fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80 + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml new file mode 100644 index 000000000000..6199be903bdb --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml @@ -0,0 +1,168 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Tue, 19 May 2020 09:27:35 GMT + x-ms-version: + - '2019-12-12' + method: HEAD + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-disposition: + - '' + content-length: + - '0' + content-md5: + - 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: + - application/octet-stream + date: + - Tue, 19 May 2020 09:27:34 GMT + etag: + - '"0x8D7FB114288CFC9"' + last-modified: + - Mon, 18 May 2020 09:53:04 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-access-tier: + - Hot + x-ms-access-tier-inferred: + - 'true' + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Mon, 18 May 2020 09:53:04 GMT + x-ms-is-current-version: + - 'true' + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-or-fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80_105f9aad-f39b-4064-8e47-ccd7937295ca: + - complete + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + x-ms-version-id: + - '2020-05-18T09:53:04.5502688Z' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Tue, 19 May 2020 09:27:35 GMT + x-ms-range: + - bytes=0-33554431 + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: "\uFEFFInvalidRangeThe\ + \ range specified is invalid for the current size of the resource.\nRequestId:af9e849b-401e-0065-20bf-2de3e4000000\n\ + Time:2020-05-19T09:27:35.8264761Z" + headers: + content-length: + - '249' + content-range: + - bytes */0 + content-type: + - application/xml + date: + - Tue, 19 May 2020 09:27:35 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-error-code: + - InvalidRange + x-ms-version: + - '2019-12-12' + status: + code: 416 + message: The range specified is invalid for the current size of the resource. +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Tue, 19 May 2020 09:27:35 GMT + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-disposition: + - '' + content-length: + - '0' + content-md5: + - 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: + - application/octet-stream + date: + - Tue, 19 May 2020 09:27:35 GMT + etag: + - '"0x8D7FB114288CFC9"' + last-modified: + - Mon, 18 May 2020 09:53:04 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Mon, 18 May 2020 09:53:04 GMT + x-ms-is-current-version: + - 'true' + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-or-fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80_105f9aad-f39b-4064-8e47-ccd7937295ca: + - complete + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + x-ms-version-id: + - '2020-05-18T09:53:04.5502688Z' + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/test_ors.py b/sdk/storage/azure-storage-blob/tests/test_ors.py new file mode 100644 index 000000000000..2a5cf8da1be0 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/test_ors.py @@ -0,0 +1,104 @@ +# coding: utf-8 + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer + +from azure.storage.blob import ( + BlobServiceClient, + BlobType, + BlobProperties, +) + +from azure.storage.blob._deserialize import deserialize_ors_policies + + +class StorageObjectReplicationTest(StorageTestCase): + SRC_CONTAINER = "test1" + DST_CONTAINER = "test2" + # BLOB_NAME = "pythonorstest" + BLOB_NAME = "bla.txt" + + # -- Test cases for Object Replication enabled account ---------------------------------------------- + # TODO the tests will temporarily use designated account, containers, and blobs to check the OR headers + + def test_deserialize_ors_policies(self): + class StubHTTPResponse: + headers = {} + + response = StubHTTPResponse() + response.headers = { + 'x-ms-or-111_111': 'Completed', + 'x-ms-or-111_222': 'Failed', + 'x-ms-or-222_111': 'Completed', + 'x-ms-or-222_222': 'Failed', + 'x-ms-or-policy-id': '333', # to be ignored + 'x-ms-not-related': 'garbage', # to be ignored + } + + result = deserialize_ors_policies(response) + self.assertEqual(len(result), 2) # 2 policies + self.assertEqual(len(result.get('111')), 2) # 2 rules for policy 111 + self.assertEqual(len(result.get('222')), 2) # 2 rules for policy 222 + + # check individual result + self.assertEqual(result.get('111').get('111'), 'Completed') + self.assertEqual(result.get('111').get('222'), 'Failed') + self.assertEqual(result.get('222').get('111'), 'Completed') + self.assertEqual(result.get('222').get('222'), 'Failed') + + @pytest.mark.playback_test_only + @GlobalStorageAccountPreparer() + def test_ors_source(self, resource_group, location, storage_account, storage_account_key): + # Arrange + bsc = BlobServiceClient( + self.account_url(storage_account, "blob"), + credential=storage_account_key) + blob = bsc.get_blob_client(container=self.SRC_CONTAINER, blob=self.BLOB_NAME) + + # Act + props = blob.get_blob_properties() + + # Assert + self.assertIsInstance(props, BlobProperties) + self.assertIsNotNone(props.object_replication_source_properties) + for policy, rule_result in props.object_replication_source_properties.items(): + self.assertNotEqual(policy, '') + self.assertIsNotNone(rule_result) + + for rule_id, result in rule_result.items(): + self.assertNotEqual(rule_id, '') + self.assertIsNotNone(result) + self.assertNotEqual(result, '') + + # Check that the download function gives back the same result + stream = blob.download_blob() + self.assertEqual(stream.properties.object_replication_source_properties, + props.object_replication_source_properties) + + @pytest.mark.playback_test_only + @GlobalStorageAccountPreparer() + def test_ors_destination(self, resource_group, location, storage_account, storage_account_key): + # Arrange + bsc = BlobServiceClient( + self.account_url(storage_account, "blob"), + credential=storage_account_key) + blob = bsc.get_blob_client(container=self.DST_CONTAINER, blob=self.BLOB_NAME) + + # Act + props = blob.get_blob_properties() + + # Assert + self.assertIsInstance(props, BlobProperties) + self.assertIsNotNone(props.object_replication_destination_policy) + + # Check that the download function gives back the same result + stream = blob.download_blob() + self.assertEqual(stream.properties.object_replication_destination_policy, + props.object_replication_destination_policy) + +# ------------------------------------------------------------------------------ From ae6d82db6d31382a27345bef3c5c4ab10e4a11a6 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 19 May 2020 02:39:24 -0700 Subject: [PATCH 2/5] Minor update to comment --- .../azure/storage/blob/_deserialize.py | 5 ++--- .../test_ors.test_ors_destination.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py index 00e0c084b729..fed85be301ce 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py @@ -19,9 +19,8 @@ def deserialize_blob_properties(response, obj, headers): - metadata = deserialize_metadata(response, obj, headers) blob_properties = BlobProperties( - metadata=metadata, + metadata=deserialize_metadata(response, obj, headers), object_replication_source_properties=deserialize_ors_policies(response), **headers ) @@ -47,7 +46,7 @@ def deserialize_ors_policies(response): policy_id = policy_and_rule_ids[0] rule_id = policy_and_rule_ids[1] - # we saw this policy already + # we are seeing this policy for the first time, so a new rule_id -> result dict is needed if parsed_result.get(policy_id) is None: parsed_result[policy_id] = {rule_id: val} else: diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml index cf4eaca92234..c80cb59b00ad 100644 --- a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml @@ -11,7 +11,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:30:18 GMT + - Tue, 19 May 2020 09:39:03 GMT x-ms-version: - '2019-12-12' method: HEAD @@ -31,7 +31,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:30:18 GMT + - Tue, 19 May 2020 09:39:03 GMT etag: - '"0x8D7FB118A463E24"' last-modified: @@ -81,7 +81,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:30:31 GMT + - Tue, 19 May 2020 09:39:04 GMT x-ms-range: - bytes=0-33554431 x-ms-version: @@ -91,8 +91,8 @@ interactions: response: body: string: "\uFEFFInvalidRangeThe\ - \ range specified is invalid for the current size of the resource.\nRequestId:cd614dd5-601e-007f-05c0-2d1559000000\n\ - Time:2020-05-19T09:30:31.3239333Z" + \ range specified is invalid for the current size of the resource.\nRequestId:1986b217-b01e-001e-0fc1-2d361a000000\n\ + Time:2020-05-19T09:39:04.3407618Z" headers: content-length: - '249' @@ -101,7 +101,7 @@ interactions: content-type: - application/xml date: - - Tue, 19 May 2020 09:30:31 GMT + - Tue, 19 May 2020 09:39:03 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-error-code: @@ -123,7 +123,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:30:31 GMT + - Tue, 19 May 2020 09:39:04 GMT x-ms-version: - '2019-12-12' method: GET @@ -143,7 +143,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:30:31 GMT + - Tue, 19 May 2020 09:39:03 GMT etag: - '"0x8D7FB118A463E24"' last-modified: From a01d662ad4026d385d6b132b18eec630eb643cbb Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 19 May 2020 02:41:12 -0700 Subject: [PATCH 3/5] Minor update to comment --- sdk/storage/azure-storage-blob/tests/test_ors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_ors.py b/sdk/storage/azure-storage-blob/tests/test_ors.py index 2a5cf8da1be0..d645fd4f052b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_ors.py +++ b/sdk/storage/azure-storage-blob/tests/test_ors.py @@ -20,12 +20,13 @@ class StorageObjectReplicationTest(StorageTestCase): SRC_CONTAINER = "test1" DST_CONTAINER = "test2" - # BLOB_NAME = "pythonorstest" BLOB_NAME = "bla.txt" # -- Test cases for Object Replication enabled account ---------------------------------------------- # TODO the tests will temporarily use designated account, containers, and blobs to check the OR headers + # TODO use generated account and set OR policy dynamically + # mock a response to test the deserializer def test_deserialize_ors_policies(self): class StubHTTPResponse: headers = {} From 0df8808e464f47df86320d6fa33f8ccfd01db02f Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 29 May 2020 14:25:21 -0700 Subject: [PATCH 4/5] Fix pylint --- .../azure-storage-blob/azure/storage/blob/_deserialize.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py index fed85be301ce..16cde64f35a4 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py @@ -96,13 +96,11 @@ def service_properties_deserialize(generated): """Deserialize a ServiceProperties objects into a dict. """ return { - 'analytics_logging': BlobAnalyticsLogging._from_generated(generated.logging), - # pylint: disable=protected-access + 'analytics_logging': BlobAnalyticsLogging._from_generated(generated.logging), # pylint: disable=protected-access 'hour_metrics': Metrics._from_generated(generated.hour_metrics), # pylint: disable=protected-access 'minute_metrics': Metrics._from_generated(generated.minute_metrics), # pylint: disable=protected-access 'cors': [CorsRule._from_generated(cors) for cors in generated.cors], # pylint: disable=protected-access 'target_version': generated.default_service_version, # pylint: disable=protected-access - 'delete_retention_policy': RetentionPolicy._from_generated(generated.delete_retention_policy), - # pylint: disable=protected-access + 'delete_retention_policy': RetentionPolicy._from_generated(generated.delete_retention_policy), # pylint: disable=protected-access 'static_website': StaticWebsite._from_generated(generated.static_website), # pylint: disable=protected-access } From 8ef4eab3184f7c0a5ddde45f72de1fbd3260918e Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Thu, 4 Jun 2020 00:35:10 -0700 Subject: [PATCH 5/5] Added async tests --- .../azure/storage/blob/_deserialize.py | 13 +- .../azure/storage/blob/_models.py | 2 +- .../test_ors.test_ors_destination.yaml | 16 +-- .../recordings/test_ors.test_ors_source.yaml | 16 +-- .../test_ors_async.test_ors_destination.yaml | 118 ++++++++++++++++++ .../test_ors_async.test_ors_source.yaml | 112 +++++++++++++++++ .../tests/test_ors_async.py | 94 ++++++++++++++ 7 files changed, 349 insertions(+), 22 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_destination.yaml create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_source.yaml create mode 100644 sdk/storage/azure-storage-blob/tests/test_ors_async.py diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py index 16cde64f35a4..738f47c22f37 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py @@ -41,16 +41,19 @@ def deserialize_ors_policies(response): parsed_result = {} + # all the ors headers have the same prefix, so we note down its length here to avoid recalculating it repeatedly + header_prefix_length = len('x-ms-or-') + for key, val in or_policy_status_headers.items(): - policy_and_rule_ids = key[len('x-ms-or-'):].split('_') + policy_and_rule_ids = key[header_prefix_length:].split('_') policy_id = policy_and_rule_ids[0] rule_id = policy_and_rule_ids[1] - # we are seeing this policy for the first time, so a new rule_id -> result dict is needed - if parsed_result.get(policy_id) is None: + try: + parsed_result[policy_id][rule_id] = val + except KeyError: + # we are seeing this policy for the first time, so a new rule_id -> result dict is needed parsed_result[policy_id] = {rule_id: val} - else: - parsed_result.get(policy_id)[rule_id] = val return parsed_result diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py index 82775a341f01..d909f38d8aea 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_models.py @@ -483,7 +483,7 @@ class BlobProperties(DictMixin): Whether this blob is encrypted. :ivar dict(str, dict(str, str)) object_replication_source_properties: Only present for blobs that have policy ids and rule ids applied to them. - Dictionary :ivar str object_replication_destination_policy: Represents the Object Replication Policy Id that created this blob. """ diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml index c80cb59b00ad..20b340386d85 100644 --- a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml @@ -11,7 +11,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:39:03 GMT + - Thu, 04 Jun 2020 07:20:14 GMT x-ms-version: - '2019-12-12' method: HEAD @@ -31,7 +31,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:39:03 GMT + - Thu, 04 Jun 2020 07:20:14 GMT etag: - '"0x8D7FB118A463E24"' last-modified: @@ -81,7 +81,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:39:04 GMT + - Thu, 04 Jun 2020 07:20:34 GMT x-ms-range: - bytes=0-33554431 x-ms-version: @@ -91,8 +91,8 @@ interactions: response: body: string: "\uFEFFInvalidRangeThe\ - \ range specified is invalid for the current size of the resource.\nRequestId:1986b217-b01e-001e-0fc1-2d361a000000\n\ - Time:2020-05-19T09:39:04.3407618Z" + \ range specified is invalid for the current size of the resource.\nRequestId:83ee5103-101e-004a-7540-3a794d000000\n\ + Time:2020-06-04T07:20:35.0604924Z" headers: content-length: - '249' @@ -101,7 +101,7 @@ interactions: content-type: - application/xml date: - - Tue, 19 May 2020 09:39:03 GMT + - Thu, 04 Jun 2020 07:20:34 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-error-code: @@ -123,7 +123,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:39:04 GMT + - Thu, 04 Jun 2020 07:20:35 GMT x-ms-version: - '2019-12-12' method: GET @@ -143,7 +143,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:39:03 GMT + - Thu, 04 Jun 2020 07:20:34 GMT etag: - '"0x8D7FB118A463E24"' last-modified: diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml index 6199be903bdb..064eaf419894 100644 --- a/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_source.yaml @@ -11,7 +11,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:27:35 GMT + - Thu, 04 Jun 2020 07:21:55 GMT x-ms-version: - '2019-12-12' method: HEAD @@ -31,7 +31,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:27:34 GMT + - Thu, 04 Jun 2020 07:21:55 GMT etag: - '"0x8D7FB114288CFC9"' last-modified: @@ -75,7 +75,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:27:35 GMT + - Thu, 04 Jun 2020 07:22:14 GMT x-ms-range: - bytes=0-33554431 x-ms-version: @@ -85,8 +85,8 @@ interactions: response: body: string: "\uFEFFInvalidRangeThe\ - \ range specified is invalid for the current size of the resource.\nRequestId:af9e849b-401e-0065-20bf-2de3e4000000\n\ - Time:2020-05-19T09:27:35.8264761Z" + \ range specified is invalid for the current size of the resource.\nRequestId:9c802dbf-401e-004a-1640-3aee2f000000\n\ + Time:2020-06-04T07:22:14.3763358Z" headers: content-length: - '249' @@ -95,7 +95,7 @@ interactions: content-type: - application/xml date: - - Tue, 19 May 2020 09:27:35 GMT + - Thu, 04 Jun 2020 07:22:13 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-error-code: @@ -117,7 +117,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) x-ms-date: - - Tue, 19 May 2020 09:27:35 GMT + - Thu, 04 Jun 2020 07:22:14 GMT x-ms-version: - '2019-12-12' method: GET @@ -137,7 +137,7 @@ interactions: content-type: - application/octet-stream date: - - Tue, 19 May 2020 09:27:35 GMT + - Thu, 04 Jun 2020 07:22:13 GMT etag: - '"0x8D7FB114288CFC9"' last-modified: diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_destination.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_destination.yaml new file mode 100644 index 000000000000..097b08a14b9e --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_destination.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: null + headers: + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:34:46 GMT + x-ms-version: + - '2019-12-12' + method: HEAD + uri: https://storagename.blob.core.windows.net/test2/bla.txt + response: + body: + string: '' + headers: + accept-ranges: bytes + content-disposition: '' + content-length: '0' + content-md5: 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: application/octet-stream + date: Thu, 04 Jun 2020 07:34:46 GMT + etag: '"0x8D7FB118A463E24"' + last-modified: Mon, 18 May 2020 09:55:04 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-access-tier: Hot + x-ms-access-tier-inferred: 'true' + x-ms-blob-type: BlockBlob + x-ms-copy-completion-time: Mon, 18 May 2020 09:55:04 GMT + x-ms-copy-id: 47d2f0e0-9739-42f5-ad74-8359dbb0c2ec + x-ms-copy-progress: 0/0 + x-ms-copy-source: https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt?versionid=2020-05-18T09:53:04.5502688Z&sv=2015-04-05&ss=b&srt=sco&sp=rwdlacup&se=2020-05-19T09%3A13%3A27.6586322Z&spr=https + x-ms-copy-status: success + x-ms-creation-time: Mon, 18 May 2020 09:55:04 GMT + x-ms-lease-state: available + x-ms-lease-status: unlocked + x-ms-or-policy-id: fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80 + x-ms-server-encrypted: 'true' + x-ms-version: '2019-12-12' + status: + code: 200 + message: OK + url: https://vbalaorcentral1.blob.core.windows.net/test2/bla.txt +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:34:47 GMT + x-ms-range: + - bytes=0-33554431 + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test2/bla.txt + response: + body: + string: "\uFEFFInvalidRangeThe\ + \ range specified is invalid for the current size of the resource.\nRequestId:4df9364d-a01e-005f-7542-3a6efe000000\n\ + Time:2020-06-04T07:34:47.3556477Z" + headers: + content-length: '249' + content-range: bytes */0 + content-type: application/xml + date: Thu, 04 Jun 2020 07:34:46 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-error-code: InvalidRange + x-ms-version: '2019-12-12' + status: + code: 416 + message: The range specified is invalid for the current size of the resource. + url: https://vbalaorcentral1.blob.core.windows.net/test2/bla.txt +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:34:47 GMT + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test2/bla.txt + response: + body: + string: '' + headers: + accept-ranges: bytes + content-disposition: '' + content-length: '0' + content-md5: 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: application/octet-stream + date: Thu, 04 Jun 2020 07:34:46 GMT + etag: '"0x8D7FB118A463E24"' + last-modified: Mon, 18 May 2020 09:55:04 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-blob-type: BlockBlob + x-ms-copy-completion-time: Mon, 18 May 2020 09:55:04 GMT + x-ms-copy-id: 47d2f0e0-9739-42f5-ad74-8359dbb0c2ec + x-ms-copy-progress: 0/0 + x-ms-copy-source: https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt?versionid=2020-05-18T09:53:04.5502688Z&sv=2015-04-05&ss=b&srt=sco&sp=rwdlacup&se=2020-05-19T09%3A13%3A27.6586322Z&spr=https + x-ms-copy-status: success + x-ms-creation-time: Mon, 18 May 2020 09:55:04 GMT + x-ms-lease-state: available + x-ms-lease-status: unlocked + x-ms-or-policy-id: fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80 + x-ms-server-encrypted: 'true' + x-ms-version: '2019-12-12' + status: + code: 200 + message: OK + url: https://vbalaorcentral1.blob.core.windows.net/test2/bla.txt +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_source.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_source.yaml new file mode 100644 index 000000000000..b49628a88338 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_ors_async.test_ors_source.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: null + headers: + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:33:24 GMT + x-ms-version: + - '2019-12-12' + method: HEAD + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: '' + headers: + accept-ranges: bytes + content-disposition: '' + content-length: '0' + content-md5: 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: application/octet-stream + date: Thu, 04 Jun 2020 07:33:24 GMT + etag: '"0x8D7FB114288CFC9"' + last-modified: Mon, 18 May 2020 09:53:04 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-access-tier: Hot + x-ms-access-tier-inferred: 'true' + x-ms-blob-type: BlockBlob + x-ms-creation-time: Mon, 18 May 2020 09:53:04 GMT + x-ms-is-current-version: 'true' + x-ms-lease-state: available + x-ms-lease-status: unlocked + x-ms-or-fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80_105f9aad-f39b-4064-8e47-ccd7937295ca: complete + x-ms-server-encrypted: 'true' + x-ms-version: '2019-12-12' + x-ms-version-id: '2020-05-18T09:53:04.5502688Z' + status: + code: 200 + message: OK + url: https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:33:24 GMT + x-ms-range: + - bytes=0-33554431 + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: "\uFEFFInvalidRangeThe\ + \ range specified is invalid for the current size of the resource.\nRequestId:c5120339-d01e-0077-6c42-3a9834000000\n\ + Time:2020-06-04T07:33:25.0346242Z" + headers: + content-length: '249' + content-range: bytes */0 + content-type: application/xml + date: Thu, 04 Jun 2020 07:33:24 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-error-code: InvalidRange + x-ms-version: '2019-12-12' + status: + code: 416 + message: The range specified is invalid for the current size of the resource. + url: https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit) + x-ms-date: + - Thu, 04 Jun 2020 07:33:25 GMT + x-ms-version: + - '2019-12-12' + method: GET + uri: https://storagename.blob.core.windows.net/test1/bla.txt + response: + body: + string: '' + headers: + accept-ranges: bytes + content-disposition: '' + content-length: '0' + content-md5: 1B2M2Y8AsgTpgAmY7PhCfg== + content-type: application/octet-stream + date: Thu, 04 Jun 2020 07:33:24 GMT + etag: '"0x8D7FB114288CFC9"' + last-modified: Mon, 18 May 2020 09:53:04 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-blob-type: BlockBlob + x-ms-creation-time: Mon, 18 May 2020 09:53:04 GMT + x-ms-is-current-version: 'true' + x-ms-lease-state: available + x-ms-lease-status: unlocked + x-ms-or-fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80_105f9aad-f39b-4064-8e47-ccd7937295ca: complete + x-ms-server-encrypted: 'true' + x-ms-version: '2019-12-12' + x-ms-version-id: '2020-05-18T09:53:04.5502688Z' + status: + code: 200 + message: OK + url: https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/test_ors_async.py b/sdk/storage/azure-storage-blob/tests/test_ors_async.py new file mode 100644 index 000000000000..c878ee863157 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/test_ors_async.py @@ -0,0 +1,94 @@ +# coding: utf-8 + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from azure.core.pipeline.transport import AioHttpTransport +from multidict import CIMultiDict, CIMultiDictProxy + +from _shared.asynctestcase import AsyncStorageTestCase +from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from azure.storage.blob import BlobProperties +from azure.storage.blob.aio import BlobServiceClient + + +# ------------------------------------------------------------------------------ +class AiohttpTestTransport(AioHttpTransport): + """Workaround to vcrpy bug: https://github.com/kevin1024/vcrpy/pull/461 + """ + + async def send(self, request, **config): + response = await super(AiohttpTestTransport, self).send(request, **config) + if not isinstance(response.headers, CIMultiDictProxy): + response.headers = CIMultiDictProxy(CIMultiDict(response.internal_response.headers)) + response.content_type = response.headers.get("content-type") + return response + + +class StorageObjectReplicationTest(StorageTestCase): + SRC_CONTAINER = "test1" + DST_CONTAINER = "test2" + BLOB_NAME = "bla.txt" + + # -- Test cases for Object Replication enabled account ---------------------------------------------- + # TODO the tests will temporarily use designated account, containers, and blobs to check the OR headers + # TODO use generated account and set OR policy dynamically + + @pytest.mark.playback_test_only + @GlobalStorageAccountPreparer() + @AsyncStorageTestCase.await_prepared_test + async def test_ors_source(self, resource_group, location, storage_account, storage_account_key): + # Arrange + bsc = BlobServiceClient( + self.account_url(storage_account, "blob"), + credential=storage_account_key, + transport=AiohttpTestTransport(connection_data_block_size=1024)) + blob = bsc.get_blob_client(container=self.SRC_CONTAINER, blob=self.BLOB_NAME) + + # Act + props = await blob.get_blob_properties() + + # Assert + self.assertIsInstance(props, BlobProperties) + self.assertIsNotNone(props.object_replication_source_properties) + for policy, rule_result in props.object_replication_source_properties.items(): + self.assertNotEqual(policy, '') + self.assertIsNotNone(rule_result) + + for rule_id, result in rule_result.items(): + self.assertNotEqual(rule_id, '') + self.assertIsNotNone(result) + self.assertNotEqual(result, '') + + # Check that the download function gives back the same result + stream = await blob.download_blob() + self.assertEqual(stream.properties.object_replication_source_properties, + props.object_replication_source_properties) + + @pytest.mark.playback_test_only + @GlobalStorageAccountPreparer() + @AsyncStorageTestCase.await_prepared_test + async def test_ors_destination(self, resource_group, location, storage_account, storage_account_key): + # Arrange + bsc = BlobServiceClient( + self.account_url(storage_account, "blob"), + credential=storage_account_key, + transport=AiohttpTestTransport(connection_data_block_size=1024)) + blob = bsc.get_blob_client(container=self.DST_CONTAINER, blob=self.BLOB_NAME) + + # Act + props = await blob.get_blob_properties() + + # Assert + self.assertIsInstance(props, BlobProperties) + self.assertIsNotNone(props.object_replication_destination_policy) + + # Check that the download function gives back the same result + stream = await blob.download_blob() + self.assertEqual(stream.properties.object_replication_destination_policy, + props.object_replication_destination_policy) + +# ------------------------------------------------------------------------------