Skip to content

Commit b68c071

Browse files
zezha-msftzezha-msft
authored
[Storage][Blob] Added support for Object Replication (#11525)
1 parent a450088 commit b68c071

File tree

9 files changed

+818
-5
lines changed

9 files changed

+818
-5
lines changed

‎sdk/storage/azure-storage-blob/CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
## 12.3.2 (Unreleased)
44

5+
**New features**
6+
7+
- Added support for object replication properties for download response and get properties.
58

69
## 12.3.1 (2020-04-29)
710

‎sdk/storage/azure-storage-blob/azure/storage/blob/_deserialize.py‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919

2020

2121
def deserialize_blob_properties(response, obj, headers):
22-
metadata = deserialize_metadata(response, obj, headers)
2322
blob_properties = BlobProperties(
24-
metadata=metadata,
23+
metadata=deserialize_metadata(response, obj, headers),
24+
object_replication_source_properties=deserialize_ors_policies(response),
2525
**headers
2626
)
2727
if 'Content-Range' in headers:
@@ -32,6 +32,32 @@ def deserialize_blob_properties(response, obj, headers):
3232
return blob_properties
3333

3434

35+
def deserialize_ors_policies(response):
36+
# For source blobs (blobs that have policy ids and rule ids applied to them),
37+
# the header will be formatted as "x-ms-or-<policy_id>_<rule_id>: {Complete, Failed}".
38+
# The value of this header is the status of the replication.
39+
or_policy_status_headers = {key: val for key, val in response.headers.items()
40+
if key.startswith('x-ms-or') and key != 'x-ms-or-policy-id'}
41+
42+
parsed_result = {}
43+
44+
# all the ors headers have the same prefix, so we note down its length here to avoid recalculating it repeatedly
45+
header_prefix_length = len('x-ms-or-')
46+
47+
for key, val in or_policy_status_headers.items():
48+
policy_and_rule_ids = key[header_prefix_length:].split('_')
49+
policy_id = policy_and_rule_ids[0]
50+
rule_id = policy_and_rule_ids[1]
51+
52+
try:
53+
parsed_result[policy_id][rule_id] = val
54+
except KeyError:
55+
# we are seeing this policy for the first time, so a new rule_id -> result dict is needed
56+
parsed_result[policy_id] = {rule_id: val}
57+
58+
return parsed_result
59+
60+
3561
def deserialize_blob_stream(response, obj, headers):
3662
blob_properties = deserialize_blob_properties(response, obj, headers)
3763
obj.properties = blob_properties
@@ -49,10 +75,10 @@ def deserialize_container_properties(response, obj, headers):
4975

5076
def get_page_ranges_result(ranges):
5177
# type: (PageList) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]
52-
page_range = [] # type: ignore
53-
clear_range = [] # type: List
78+
page_range = [] # type: ignore
79+
clear_range = [] # type: List
5480
if ranges.page_range:
55-
page_range = [{'start': b.start, 'end': b.end} for b in ranges.page_range] # type: ignore
81+
page_range = [{'start': b.start, 'end': b.end} for b in ranges.page_range] # type: ignore
5682
if ranges.clear_range:
5783
clear_range = [{'start': b.start, 'end': b.end} for b in ranges.clear_range]
5884
return page_range, clear_range # type: ignore

‎sdk/storage/azure-storage-blob/azure/storage/blob/_models.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,11 @@ class BlobProperties(DictMixin):
481481
container-level scope is configured to allow overrides. Otherwise an error will be raised.
482482
:ivar bool request_server_encrypted:
483483
Whether this blob is encrypted.
484+
:ivar dict(str, dict(str, str)) object_replication_source_properties:
485+
Only present for blobs that have policy ids and rule ids applied to them.
486+
Dictionary<policy_id, Dictionary<rule_id, status of replication(complete,failed)>
487+
:ivar str object_replication_destination_policy:
488+
Represents the Object Replication Policy Id that created this blob.
484489
"""
485490

486491
def __init__(self, **kwargs):
@@ -511,6 +516,8 @@ def __init__(self, **kwargs):
511516
self.encryption_key_sha256 = kwargs.get('x-ms-encryption-key-sha256')
512517
self.encryption_scope = kwargs.get('x-ms-encryption-scope')
513518
self.request_server_encrypted = kwargs.get('x-ms-server-encrypted')
519+
self.object_replication_source_properties = kwargs.get('object_replication_source_properties')
520+
self.object_replication_destination_policy = kwargs.get('x-ms-or-policy-id')
514521

515522
@classmethod
516523
def _from_generated(cls, generated):

‎sdk/storage/azure-storage-blob/tests/recordings/test_ors.test_ors_destination.yaml‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
Accept:
6+
- '*/*'
7+
Accept-Encoding:
8+
- gzip, deflate
9+
Connection:
10+
- keep-alive
11+
User-Agent:
12+
- azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit)
13+
x-ms-date:
14+
- Thu, 04 Jun 2020 07:20:14 GMT
15+
x-ms-version:
16+
- '2019-12-12'
17+
method: HEAD
18+
uri: https://storagename.blob.core.windows.net/test2/bla.txt
19+
response:
20+
body:
21+
string: ''
22+
headers:
23+
accept-ranges:
24+
- bytes
25+
content-disposition:
26+
- ''
27+
content-length:
28+
- '0'
29+
content-md5:
30+
- 1B2M2Y8AsgTpgAmY7PhCfg==
31+
content-type:
32+
- application/octet-stream
33+
date:
34+
- Thu, 04 Jun 2020 07:20:14 GMT
35+
etag:
36+
- '"0x8D7FB118A463E24"'
37+
last-modified:
38+
- Mon, 18 May 2020 09:55:04 GMT
39+
server:
40+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
41+
x-ms-access-tier:
42+
- Hot
43+
x-ms-access-tier-inferred:
44+
- 'true'
45+
x-ms-blob-type:
46+
- BlockBlob
47+
x-ms-copy-completion-time:
48+
- Mon, 18 May 2020 09:55:04 GMT
49+
x-ms-copy-id:
50+
- 47d2f0e0-9739-42f5-ad74-8359dbb0c2ec
51+
x-ms-copy-progress:
52+
- 0/0
53+
x-ms-copy-source:
54+
- https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt?versionid=2020-05-18T09:53:04.5502688Z&sv=2015-04-05&ss=b&srt=sco&sp=rwdlacup&se=2020-05-19T09%3A13%3A27.6586322Z&spr=https
55+
x-ms-copy-status:
56+
- success
57+
x-ms-creation-time:
58+
- Mon, 18 May 2020 09:55:04 GMT
59+
x-ms-lease-state:
60+
- available
61+
x-ms-lease-status:
62+
- unlocked
63+
x-ms-or-policy-id:
64+
- fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80
65+
x-ms-server-encrypted:
66+
- 'true'
67+
x-ms-version:
68+
- '2019-12-12'
69+
status:
70+
code: 200
71+
message: OK
72+
- request:
73+
body: null
74+
headers:
75+
Accept:
76+
- application/xml
77+
Accept-Encoding:
78+
- gzip, deflate
79+
Connection:
80+
- keep-alive
81+
User-Agent:
82+
- azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit)
83+
x-ms-date:
84+
- Thu, 04 Jun 2020 07:20:34 GMT
85+
x-ms-range:
86+
- bytes=0-33554431
87+
x-ms-version:
88+
- '2019-12-12'
89+
method: GET
90+
uri: https://storagename.blob.core.windows.net/test2/bla.txt
91+
response:
92+
body:
93+
string: "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><Error><Code>InvalidRange</Code><Message>The\
94+
\ range specified is invalid for the current size of the resource.\nRequestId:83ee5103-101e-004a-7540-3a794d000000\n\
95+
Time:2020-06-04T07:20:35.0604924Z</Message></Error>"
96+
headers:
97+
content-length:
98+
- '249'
99+
content-range:
100+
- bytes */0
101+
content-type:
102+
- application/xml
103+
date:
104+
- Thu, 04 Jun 2020 07:20:34 GMT
105+
server:
106+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
107+
x-ms-error-code:
108+
- InvalidRange
109+
x-ms-version:
110+
- '2019-12-12'
111+
status:
112+
code: 416
113+
message: The range specified is invalid for the current size of the resource.
114+
- request:
115+
body: null
116+
headers:
117+
Accept:
118+
- application/xml
119+
Accept-Encoding:
120+
- gzip, deflate
121+
Connection:
122+
- keep-alive
123+
User-Agent:
124+
- azsdk-python-storage-blob/12.3.2 Python/3.7.4 (Darwin-19.4.0-x86_64-i386-64bit)
125+
x-ms-date:
126+
- Thu, 04 Jun 2020 07:20:35 GMT
127+
x-ms-version:
128+
- '2019-12-12'
129+
method: GET
130+
uri: https://storagename.blob.core.windows.net/test2/bla.txt
131+
response:
132+
body:
133+
string: ''
134+
headers:
135+
accept-ranges:
136+
- bytes
137+
content-disposition:
138+
- ''
139+
content-length:
140+
- '0'
141+
content-md5:
142+
- 1B2M2Y8AsgTpgAmY7PhCfg==
143+
content-type:
144+
- application/octet-stream
145+
date:
146+
- Thu, 04 Jun 2020 07:20:34 GMT
147+
etag:
148+
- '"0x8D7FB118A463E24"'
149+
last-modified:
150+
- Mon, 18 May 2020 09:55:04 GMT
151+
server:
152+
- Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
153+
x-ms-blob-type:
154+
- BlockBlob
155+
x-ms-copy-completion-time:
156+
- Mon, 18 May 2020 09:55:04 GMT
157+
x-ms-copy-id:
158+
- 47d2f0e0-9739-42f5-ad74-8359dbb0c2ec
159+
x-ms-copy-progress:
160+
- 0/0
161+
x-ms-copy-source:
162+
- https://ortestsaccountcbn1.blob.core.windows.net/test1/bla.txt?versionid=2020-05-18T09:53:04.5502688Z&sv=2015-04-05&ss=b&srt=sco&sp=rwdlacup&se=2020-05-19T09%3A13%3A27.6586322Z&spr=https
163+
x-ms-copy-status:
164+
- success
165+
x-ms-creation-time:
166+
- Mon, 18 May 2020 09:55:04 GMT
167+
x-ms-lease-state:
168+
- available
169+
x-ms-lease-status:
170+
- unlocked
171+
x-ms-or-policy-id:
172+
- fd2da1b9-56f5-45ff-9eb6-310e6dfc2c80
173+
x-ms-server-encrypted:
174+
- 'true'
175+
x-ms-version:
176+
- '2019-12-12'
177+
status:
178+
code: 200
179+
message: OK
180+
version: 1

0 commit comments

Comments
 (0)