Propagate new sorting header sorting to other packages, add in explic…

…it test for previously failing test case in Blob

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-blob",
-  "Tag": "python/storage/azure-storage-blob_ba57356b0e"
+  "Tag": "python/storage/azure-storage-blob_f9158b580e"
 }

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py

@@ -8,6 +8,7 @@
 import re
 from typing import List, Tuple
 from urllib.parse import unquote, urlparse
+from functools import cmp_to_key
 
 try:
     from yarl import URL
@@ -27,6 +28,66 @@
 logger = logging.getLogger(__name__)
 
 
+table_lv0 = [
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
+    0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0x0, 0xd1a, 0xd1c, 0xd1e,
+    0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
+    0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
+    0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
+    0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
+    0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
+]
+
+table_lv4 = [
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+]
+
+def compare(lhs: str, rhs: str) -> int:  # pylint:disable=too-many-return-statements
+    tables = [table_lv0, table_lv4]
+    curr_level, i, j, n = 0, 0, 0, len(tables)
+    lhs_len = len(lhs)
+    rhs_len = len(rhs)
+    while curr_level < n:
+        if curr_level == (n - 1) and i != j:
+            if i > j:
+                return -1
+            if i < j:
+                return 1
+            return 0
+
+        w1 = tables[curr_level][ord(lhs[i])] if i < lhs_len else 0x1
+        w2 = tables[curr_level][ord(rhs[j])] if j < rhs_len else 0x1
+
+        if w1 == 0x1 and w2 == 0x1:
+            i = 0
+            j = 0
+            curr_level += 1
+        elif w1 == w2:
+            i += 1
+            j += 1
+        elif w1 == 0:
+            i += 1
+        elif w2 == 0:
+            j += 1
+        else:
+            if w1 < w2:
+                return -1
+            if w1 > w2:
+                return 1
+            return 0
+    return 1
+
+
 # wraps a given exception with the desired exception type
 def _wrap_exception(ex, desired_type):
     msg = ""
@@ -36,8 +97,6 @@ def _wrap_exception(ex, desired_type):
 
 # This method attempts to emulate the sorting done by the service
 def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-    # Define the custom alphabet for weights
-    custom_weights = "-!#$%&*.^_|~+\"\'(),/`~0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]abcdefghijklmnopqrstuvwxyz{}"
 
     # Build dict of tuples and list of keys
     header_dict = {}
@@ -46,9 +105,8 @@ def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str
         header_dict[k] = v
         header_keys.append(k)
 
-    # Sort according to custom defined weights
     try:
-        header_keys = sorted(header_keys, key=lambda word: [custom_weights.index(c) for c in word])
+        header_keys = sorted(header_keys, key=cmp_to_key(compare))
     except ValueError as exc:
         raise ValueError("Illegal character encountered when sorting headers.") from exc
 

sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py

@@ -3124,4 +3124,45 @@ def test_header_metadata_sort_in_upload_blob(self, **kwargs):
         # Act
         blob_client.upload_blob(data, length=len(data), metadata=metadata)
 
+    @BlobPreparer()
+    @recorded_by_proxy
+    def test_header_metadata_sort_in_upload_blob_translation(self, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+        storage_account_key = kwargs.pop("storage_account_key")
+
+        self._setup()
+        data = b'hello world'
+        bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), storage_account_key)
+        try:
+            container_client = bsc.create_container(self.container_name)
+        except:
+            container_client = bsc.get_container_client(self.container_name)
+        blob_client = container_client.get_blob_client('blob1')
+
+        # Hand-picked metadata examples that sorted incorrectly with our previous implementation.
+        metadata = {
+            'test': 'val',
+            'test-': 'val',
+            'test--': 'val',
+            'test-_': 'val',
+            'test_-': 'val',
+            'test__': 'val',
+            'test-a': 'val',
+            'test-A': 'val',
+            'test-_A': 'val',
+            'test_a': 'val',
+            'test_Z': 'val',
+            'test_a_': 'val',
+            'test_a-': 'val',
+            'test_a-_': 'val',
+        }
+
+        # Act
+        # If we hit invalid metadata error, that means we have successfully sorted headers properly to pass auth error
+        with pytest.raises(HttpResponseError) as e:
+            blob_client.upload_blob(data, length=len(data), metadata=metadata)
+
+        # Assert
+        assert StorageErrorCode.invalid_metadata == e.value.error_code
+
 # ------------------------------------------------------------------------------

sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py

@@ -3092,4 +3092,69 @@ async def test_header_metadata_sort_in_upload_blob(self, **kwargs):
         # Act
         await blob_client.upload_blob(data, length=len(data), metadata=metadata)
 
+    @BlobPreparer()
+    @recorded_by_proxy_async
+    async def test_header_metadata_sort_in_upload_blob(self, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+        storage_account_key = kwargs.pop("storage_account_key")
+
+        self._setup()
+        data = b'hello world'
+        bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), storage_account_key)
+        try:
+            container_client = await bsc.create_container(self.container_name)
+        except:
+            container_client = bsc.get_container_client(self.container_name)
+        blob_client = container_client.get_blob_client('blob1')
+
+        # Hand-picked metadata examples as Python & service don't sort '_' with the same weight
+        metadata = {'a0': 'a', 'a1': 'a', 'a2': 'a', 'a3': 'a', 'a4': 'a', 'a5': 'a', 'a6': 'a', 'a7': 'a', 'a8': 'a',
+                    'a9': 'a', '_': 'a', '_a': 'a', 'a_': 'a', '__': 'a', '_a_': 'a', 'b': 'a', 'c': 'a', 'y': 'a',
+                    'z': 'z_', '_z': 'a', '_F': 'a', 'F': 'a', 'F_': 'a', '_F_': 'a', '__F': 'a', '__a': 'a', 'a__': 'a'
+                    }
+
+        # Act
+        await blob_client.upload_blob(data, length=len(data), metadata=metadata)
+
+    @BlobPreparer()
+    @recorded_by_proxy_async
+    async def test_header_metadata_sort_in_upload_blob_translation(self, **kwargs):
+        storage_account_name = kwargs.pop("storage_account_name")
+        storage_account_key = kwargs.pop("storage_account_key")
+
+        self._setup()
+        data = b'hello world'
+        bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), storage_account_key)
+        try:
+            container_client = await bsc.create_container(self.container_name)
+        except:
+            container_client = bsc.get_container_client(self.container_name)
+        blob_client = container_client.get_blob_client('blob1')
+
+        # Hand-picked metadata examples that sorted incorrectly with our previous implementation.
+        metadata = {
+            'test': 'val',
+            'test-': 'val',
+            'test--': 'val',
+            'test-_': 'val',
+            'test_-': 'val',
+            'test__': 'val',
+            'test-a': 'val',
+            'test-A': 'val',
+            'test-_A': 'val',
+            'test_a': 'val',
+            'test_Z': 'val',
+            'test_a_': 'val',
+            'test_a-': 'val',
+            'test_a-_': 'val',
+        }
+
+        # Act
+        # If we hit invalid metadata error, that means we have successfully sorted headers properly to pass auth error
+        with pytest.raises(HttpResponseError) as e:
+            await blob_client.upload_blob(data, length=len(data), metadata=metadata)
+
+        # Assert
+        assert StorageErrorCode.invalid_metadata == e.value.error_code
+
 # ------------------------------------------------------------------------------

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/authentication.py

@@ -8,6 +8,7 @@
 import re
 from typing import List, Tuple
 from urllib.parse import unquote, urlparse
+from functools import cmp_to_key
 
 try:
     from yarl import URL
@@ -27,6 +28,66 @@
 logger = logging.getLogger(__name__)
 
 
+table_lv0 = [
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
+    0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0x0, 0xd1a, 0xd1c, 0xd1e,
+    0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
+    0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
+    0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
+    0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
+    0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
+]
+
+table_lv4 = [
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+]
+
+def compare(lhs: str, rhs: str) -> int:  # pylint:disable=too-many-return-statements
+    tables = [table_lv0, table_lv4]
+    curr_level, i, j, n = 0, 0, 0, len(tables)
+    lhs_len = len(lhs)
+    rhs_len = len(rhs)
+    while curr_level < n:
+        if curr_level == (n - 1) and i != j:
+            if i > j:
+                return -1
+            if i < j:
+                return 1
+            return 0
+
+        w1 = tables[curr_level][ord(lhs[i])] if i < lhs_len else 0x1
+        w2 = tables[curr_level][ord(rhs[j])] if j < rhs_len else 0x1
+
+        if w1 == 0x1 and w2 == 0x1:
+            i = 0
+            j = 0
+            curr_level += 1
+        elif w1 == w2:
+            i += 1
+            j += 1
+        elif w1 == 0:
+            i += 1
+        elif w2 == 0:
+            j += 1
+        else:
+            if w1 < w2:
+                return -1
+            if w1 > w2:
+                return 1
+            return 0
+    return 1
+
+
 # wraps a given exception with the desired exception type
 def _wrap_exception(ex, desired_type):
     msg = ""
@@ -36,8 +97,6 @@ def _wrap_exception(ex, desired_type):
 
 # This method attempts to emulate the sorting done by the service
 def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-    # Define the custom alphabet for weights
-    custom_weights = "-!#$%&*.^_|~+\"\'(),/`~0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]abcdefghijklmnopqrstuvwxyz{}"
 
     # Build dict of tuples and list of keys
     header_dict = {}
@@ -46,9 +105,8 @@ def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str
         header_dict[k] = v
         header_keys.append(k)
 
-    # Sort according to custom defined weights
     try:
-        header_keys = sorted(header_keys, key=lambda word: [custom_weights.index(c) for c in word])
+        header_keys = sorted(header_keys, key=cmp_to_key(compare))
     except ValueError as exc:
         raise ValueError("Illegal character encountered when sorting headers.") from exc
 

sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/authentication.py

@@ -40,17 +40,6 @@
     0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
 ]
 
-table_lv2 = [
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12,
-    0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-    0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-]
-
 table_lv4 = [
     0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
     0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
@@ -62,9 +51,11 @@
     0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 ]
 
-def compare(lhs: str, rhs: str):  # pylint:disable=too-many-return-statements
-    tables = [table_lv0, table_lv2, table_lv4]
+def compare(lhs: str, rhs: str) -> int:  # pylint:disable=too-many-return-statements
+    tables = [table_lv0, table_lv4]
     curr_level, i, j, n = 0, 0, 0, len(tables)
+    lhs_len = len(lhs)
+    rhs_len = len(rhs)
     while curr_level < n:
         if curr_level == (n - 1) and i != j:
             if i > j:
@@ -73,11 +64,12 @@ def compare(lhs: str, rhs: str):  # pylint:disable=too-many-return-statements
                 return 1
             return 0
 
-        w1 = tables[curr_level][ord(lhs[i])] if i < len(lhs) else 0x1
-        w2 = tables[curr_level][ord(rhs[j])] if j < len(rhs) else 0x1
+        w1 = tables[curr_level][ord(lhs[i])] if i < lhs_len else 0x1
+        w2 = tables[curr_level][ord(rhs[j])] if j < rhs_len else 0x1
 
         if w1 == 0x1 and w2 == 0x1:
-            i, j = 0, 0
+            i = 0
+            j = 0
             curr_level += 1
         elif w1 == w2:
             i += 1
@@ -104,7 +96,6 @@ def _wrap_exception(ex, desired_type):
 
 # This method attempts to emulate the sorting done by the service
 def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
-    # Define the custom alphabet for weights
 
     # Build dict of tuples and list of keys
     header_dict = {}
@@ -113,9 +104,7 @@ def _storage_header_sort(input_headers: List[Tuple[str, str]]) -> List[Tuple[str
         header_dict[k] = v
         header_keys.append(k)
 
-    # Sort according to custom defined weights
     try:
-        # header_keys = sorted(header_keys, key=lambda word: [custom_weights.index(c) for c in word])
         header_keys = sorted(header_keys, key=cmp_to_key(compare))
     except ValueError as exc:
         raise ValueError("Illegal character encountered when sorting headers.") from exc