[Storage] [STG94] Support more service versions in Queue, propagate S…

…torageBearer Challenge to all packages (#35356)

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py

@@ -231,7 +231,7 @@ def _create_pipeline(
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = StorageBearerTokenCredentialPolicy(credential, audience)
+            self._credential_policy = StorageBearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py

@@ -6,7 +6,7 @@
 # mypy: disable-error-code="attr-defined"
 
 import logging
-from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING, Union
+from typing import Any, cast, Dict, Optional, Tuple, TYPE_CHECKING, Union
 
 from azure.core.async_paging import AsyncList
 from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential
@@ -104,7 +104,8 @@ def _create_pipeline(
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = AsyncStorageBearerTokenCredentialPolicy(credential, audience)
+            self._credential_policy = AsyncStorageBearerTokenCredentialPolicy(
+                                        cast(AsyncTokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-file-datalake/CHANGELOG.md

@@ -3,7 +3,7 @@
 ## 12.16.0b1 (Unreleased)
 
 ### Features Added
-
+- Updated OAuth implementation to use the AAD scope returned in a Bearer challenge.
 
 ## 12.15.0 (Unreleased)
 

sdk/storage/azure-storage-file-datalake/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-file-datalake",
-  "Tag": "python/storage/azure-storage-file-datalake_922696d4ec"
+  "Tag": "python/storage/azure-storage-file-datalake_dec64d97f2"
 }

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py

@@ -22,6 +22,7 @@
     '2020-06-12',
     '2020-08-04',
     '2020-10-02',
+    '2020-12-06',
     '2021-02-12',
     '2021-04-10',
     '2021-06-08',

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py

@@ -23,7 +23,6 @@
 from azure.core.pipeline.transport import HttpTransport, RequestsTransport  # pylint: disable=non-abstract-transport-import, no-name-in-module
 from azure.core.pipeline.policies import (
     AzureSasCredentialPolicy,
-    BearerTokenCredentialPolicy,
     ContentDecodePolicy,
     DistributedTracingPolicy,
     HttpLoggingPolicy,
@@ -38,6 +37,7 @@
 from .policies import (
     ExponentialRetry,
     QueueMessagePolicy,
+    StorageBearerTokenCredentialPolicy,
     StorageContentValidation,
     StorageHeadersPolicy,
     StorageHosts,
@@ -231,7 +231,7 @@ def _create_pipeline(
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = BearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
+            self._credential_policy = StorageBearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py

@@ -14,7 +14,6 @@
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline import AsyncPipeline
 from azure.core.pipeline.policies import (
-    AsyncBearerTokenCredentialPolicy,
     AsyncRedirectPolicy,
     AzureSasCredentialPolicy,
     ContentDecodePolicy,
@@ -34,7 +33,7 @@
     StorageHosts,
     StorageRequestHook,
 )
-from .policies_async import AsyncStorageResponseHook
+from .policies_async import AsyncStorageBearerTokenCredentialPolicy, AsyncStorageResponseHook
 from .response_handlers import PartialBatchErrorException, process_storage_error
 from .._shared_access_signature import _is_credential_sastoken
 
@@ -97,15 +96,16 @@ def _create_pipeline(
         **kwargs: Any
     ) -> Tuple[StorageConfiguration, AsyncPipeline]:
         self._credential_policy: Optional[
-            Union[AsyncBearerTokenCredentialPolicy,
+            Union[AsyncStorageBearerTokenCredentialPolicy,
             SharedKeyCredentialPolicy,
             AzureSasCredentialPolicy]] = None
         if hasattr(credential, 'get_token'):
             if kwargs.get('audience'):
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = AsyncBearerTokenCredentialPolicy(cast(AsyncTokenCredential, credential), audience)
+            self._credential_policy = AsyncStorageBearerTokenCredentialPolicy(
+                                        cast(AsyncTokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py

@@ -495,7 +495,6 @@ def test_bad_audience_service_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            dsc.list_file_systems()
-            dsc.create_file_system('testfs22')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        dsc.list_file_systems()
+        dsc.create_file_system('testfs22')

sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py

@@ -497,7 +497,6 @@ async def test_bad_audience_service_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            dsc.list_file_systems()
-            await dsc.create_file_system(file_system_name + '1')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        dsc.list_file_systems()
+        await dsc.create_file_system(file_system_name + '1')

sdk/storage/azure-storage-file-datalake/tests/test_directory.py

@@ -1603,10 +1603,9 @@ def test_bad_audience_dir_client(self, **kwargs):
             credential=token_credential, audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            directory_client.exists()
-            directory_client.create_sub_directory('testsubdir')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        directory_client.exists()
+        directory_client.create_sub_directory('testsubdir')
 
 # ------------------------------------------------------------------------------
 if __name__ == '__main__':

sdk/storage/azure-storage-file-datalake/tests/test_directory_async.py

@@ -1553,10 +1553,9 @@ async def test_bad_audience_dir_client(self, **kwargs):
             credential=token_credential, audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            await directory_client.exists()
-            await directory_client.create_sub_directory('testsubdir')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        await directory_client.exists()
+        await directory_client.create_sub_directory('testsubdir')
 
 # ------------------------------------------------------------------------------
 if __name__ == '__main__':

sdk/storage/azure-storage-file-datalake/tests/test_file.py

@@ -1628,11 +1628,10 @@ def test_bad_audience_file_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
         data = b'Hello world'
-        with pytest.raises(ClientAuthenticationError):
-            fc.get_file_properties()
-            fc.upload_data(data, overwrite=True)
+        fc.get_file_properties()
+        fc.upload_data(data, overwrite=True)
 
 
 # ------------------------------------------------------------------------------

sdk/storage/azure-storage-file-datalake/tests/test_file_async.py

@@ -1526,11 +1526,10 @@ async def test_bad_audience_file_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
         data = b'Hello world'
-        with pytest.raises(ClientAuthenticationError):
-            await fc.get_file_properties()
-            await fc.upload_data(data, overwrite=True)
+        await fc.get_file_properties()
+        await fc.upload_data(data, overwrite=True)
 
 
 # ------------------------------------------------------------------------------

sdk/storage/azure-storage-file-datalake/tests/test_file_system.py

@@ -1121,10 +1121,9 @@ def test_bad_audience_service_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            fsc.exists()
-            fsc.create_directory('testdir22')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        fsc.exists()
+        fsc.create_directory('testdir22')
 
 # ------------------------------------------------------------------------------
 if __name__ == '__main__':

sdk/storage/azure-storage-file-datalake/tests/test_file_system_async.py

@@ -1251,10 +1251,9 @@ async def test_bad_audience_service_client(self, **kwargs):
             audience=f'https://badaudience.blob.core.windows.net/'
         )
 
-        # Assert
-        with pytest.raises(ClientAuthenticationError):
-            await fsc.exists()
-            await fsc.create_directory('testdir22')
+        # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge
+        await fsc.exists()
+        await fsc.create_directory('testdir22')
 
 # ------------------------------------------------------------------------------
 if __name__ == '__main__':

sdk/storage/azure-storage-file-share/azure/storage/fileshare/_serialize.py

@@ -30,6 +30,7 @@
     '2020-06-12',
     '2020-08-04',
     '2020-10-02',
+    '2020-12-06',
     '2021-02-12',
     '2021-04-10',
     '2021-06-08',

sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client.py

@@ -23,7 +23,6 @@
 from azure.core.pipeline.transport import HttpTransport, RequestsTransport  # pylint: disable=non-abstract-transport-import, no-name-in-module
 from azure.core.pipeline.policies import (
     AzureSasCredentialPolicy,
-    BearerTokenCredentialPolicy,
     ContentDecodePolicy,
     DistributedTracingPolicy,
     HttpLoggingPolicy,
@@ -38,6 +37,7 @@
 from .policies import (
     ExponentialRetry,
     QueueMessagePolicy,
+    StorageBearerTokenCredentialPolicy,
     StorageContentValidation,
     StorageHeadersPolicy,
     StorageHosts,
@@ -231,7 +231,7 @@ def _create_pipeline(
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = BearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
+            self._credential_policy = StorageBearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client_async.py

@@ -14,7 +14,6 @@
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline import AsyncPipeline
 from azure.core.pipeline.policies import (
-    AsyncBearerTokenCredentialPolicy,
     AsyncRedirectPolicy,
     AzureSasCredentialPolicy,
     ContentDecodePolicy,
@@ -34,7 +33,7 @@
     StorageHosts,
     StorageRequestHook,
 )
-from .policies_async import AsyncStorageResponseHook
+from .policies_async import AsyncStorageBearerTokenCredentialPolicy, AsyncStorageResponseHook
 from .response_handlers import PartialBatchErrorException, process_storage_error
 from .._shared_access_signature import _is_credential_sastoken
 
@@ -97,15 +96,16 @@ def _create_pipeline(
         **kwargs: Any
     ) -> Tuple[StorageConfiguration, AsyncPipeline]:
         self._credential_policy: Optional[
-            Union[AsyncBearerTokenCredentialPolicy,
+            Union[AsyncStorageBearerTokenCredentialPolicy,
             SharedKeyCredentialPolicy,
             AzureSasCredentialPolicy]] = None
         if hasattr(credential, 'get_token'):
             if kwargs.get('audience'):
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = AsyncBearerTokenCredentialPolicy(cast(AsyncTokenCredential, credential), audience)
+            self._credential_policy = AsyncStorageBearerTokenCredentialPolicy(
+                                        cast(AsyncTokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-file-share/tests/test_file.py

@@ -3762,6 +3762,6 @@ def test_bad_audience_file_client(self, **kwargs):
 
         # Assert
         with pytest.raises(ClientAuthenticationError):
-            file_client.get_file_properties()
+            file_client.exists()
 
 # ------------------------------------------------------------------------------

sdk/storage/azure-storage-file-share/tests/test_file_async.py

@@ -3878,4 +3878,4 @@ async def test_bad_audience_file_client(self, **kwargs):
 
         # Assert
         with pytest.raises(ClientAuthenticationError):
-            await file_client.get_file_properties()
+            await file_client.exists()

sdk/storage/azure-storage-queue/CHANGELOG.md

@@ -3,7 +3,7 @@
 ## 12.11.0b1 (Unreleased)
 
 ### Features Added
-
+- Updated OAuth implementation to use the AAD scope returned in a Bearer challenge.
 
 ## 12.10.0 (Unreleased)
 

sdk/storage/azure-storage-queue/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-queue",
-  "Tag": "python/storage/azure-storage-queue_effe33bc27"
+  "Tag": "python/storage/azure-storage-queue_8161cc758c"
 }

sdk/storage/azure-storage-queue/azure/storage/queue/_serialize.py

@@ -15,7 +15,18 @@
     '2020-06-12',
     '2020-08-04',
     '2020-10-02',
+    '2020-12-06',
     '2021-02-12',
+    '2021-04-10',
+    '2021-06-08',
+    '2021-08-06',
+    '2021-12-02',
+    '2022-11-02',
+    '2023-01-03',
+    '2023-05-03',
+    '2023-08-03',
+    '2023-11-03',
+    '2024-05-04',
     '2024-08-04',
 ]
 

sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py

@@ -23,7 +23,6 @@
 from azure.core.pipeline.transport import HttpTransport, RequestsTransport  # pylint: disable=non-abstract-transport-import, no-name-in-module
 from azure.core.pipeline.policies import (
     AzureSasCredentialPolicy,
-    BearerTokenCredentialPolicy,
     ContentDecodePolicy,
     DistributedTracingPolicy,
     HttpLoggingPolicy,
@@ -38,6 +37,7 @@
 from .policies import (
     ExponentialRetry,
     QueueMessagePolicy,
+    StorageBearerTokenCredentialPolicy,
     StorageContentValidation,
     StorageHeadersPolicy,
     StorageHosts,
@@ -231,7 +231,7 @@ def _create_pipeline(
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = BearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
+            self._credential_policy = StorageBearerTokenCredentialPolicy(cast(TokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):

sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client_async.py

@@ -14,7 +14,6 @@
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline import AsyncPipeline
 from azure.core.pipeline.policies import (
-    AsyncBearerTokenCredentialPolicy,
     AsyncRedirectPolicy,
     AzureSasCredentialPolicy,
     ContentDecodePolicy,
@@ -34,7 +33,7 @@
     StorageHosts,
     StorageRequestHook,
 )
-from .policies_async import AsyncStorageResponseHook
+from .policies_async import AsyncStorageBearerTokenCredentialPolicy, AsyncStorageResponseHook
 from .response_handlers import PartialBatchErrorException, process_storage_error
 from .._shared_access_signature import _is_credential_sastoken
 
@@ -97,15 +96,16 @@ def _create_pipeline(
         **kwargs: Any
     ) -> Tuple[StorageConfiguration, AsyncPipeline]:
         self._credential_policy: Optional[
-            Union[AsyncBearerTokenCredentialPolicy,
+            Union[AsyncStorageBearerTokenCredentialPolicy,
             SharedKeyCredentialPolicy,
             AzureSasCredentialPolicy]] = None
         if hasattr(credential, 'get_token'):
             if kwargs.get('audience'):
                 audience = str(kwargs.pop('audience')).rstrip('/') + DEFAULT_OAUTH_SCOPE
             else:
                 audience = STORAGE_OAUTH_SCOPE
-            self._credential_policy = AsyncBearerTokenCredentialPolicy(cast(AsyncTokenCredential, credential), audience)
+            self._credential_policy = AsyncStorageBearerTokenCredentialPolicy(
+                                        cast(AsyncTokenCredential, credential), audience)
         elif isinstance(credential, SharedKeyCredentialPolicy):
             self._credential_policy = credential
         elif isinstance(credential, AzureSasCredential):