Jumbo blob support

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -94,7 +94,7 @@ class BlobClient(StorageAccountHostsMixin):  # pylint: disable=too-many-public-m
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py

@@ -77,7 +77,7 @@ class BlobServiceClient(StorageAccountHostsMixin):
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py

@@ -97,7 +97,7 @@ class ContainerClient(StorageAccountHostsMixin):
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/version.py

@@ -9,5 +9,5 @@
 # regenerated.
 # --------------------------------------------------------------------------
 
-VERSION = "2019-10-10"
+VERSION = "2019-12-12"
 

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py

@@ -246,6 +246,8 @@ def _create_pipeline(self, credential, **kwargs):
             DistributedTracingPolicy(**kwargs),
             HttpLoggingPolicy(**kwargs)
         ]
+        if kwargs.get("_additional_pipeline_policies"):
+            policies = policies + kwargs.get("_additional_pipeline_policies")
         return config, Pipeline(config.transport, policies=policies)
 
     def _batch_send(

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py

@@ -102,6 +102,8 @@ def _create_pipeline(self, credential, **kwargs):
             DistributedTracingPolicy(**kwargs),
             HttpLoggingPolicy(**kwargs),
         ]
+        if kwargs.get("_additional_pipeline_policies"):
+            policies = policies + kwargs.get("_additional_pipeline_policies")
         return config, AsyncPipeline(config.transport, policies=policies)
 
     async def _batch_send(

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/constants.py

@@ -17,9 +17,10 @@
 # for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned)
 # The socket timeout is now the maximum total duration to send all data.
 if sys.version_info >= (3, 5):
-    # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds
-    # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed)
-    READ_TIMEOUT = 2000
+    # the timeout to connect is 20 seconds, and the read timeout is 80000 seconds
+    # the 80000 seconds was calculated with:
+    # 4000MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed)
+    READ_TIMEOUT = 80000
 
 STORAGE_OAUTH_SCOPE = "https://storage.azure.com/.default"
 

sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py

@@ -80,8 +80,8 @@ def upload_block_blob(  # pylint: disable=too-many-locals
         blob_headers = kwargs.pop('blob_headers', None)
         tier = kwargs.pop('standard_blob_tier', None)
 
-        # Do single put if the size is smaller than config.max_single_put_size
-        if adjusted_count is not None and (adjusted_count < blob_settings.max_single_put_size):
+        # Do single put if the size is smaller than or equal config.max_single_put_size
+        if adjusted_count is not None and (adjusted_count <= blob_settings.max_single_put_size):
             try:
                 data = data.read(length)
                 if not isinstance(data, six.binary_type):

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py

@@ -74,7 +74,7 @@ class BlobClient(AsyncStorageAccountHostsMixin, BlobClientBase):  # pylint: disa
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py

@@ -76,7 +76,7 @@ class BlobServiceClient(AsyncStorageAccountHostsMixin, BlobServiceClientBase):
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py

@@ -79,7 +79,7 @@ class ContainerClient(AsyncStorageAccountHostsMixin, ContainerClientBase):
         The hostname of the secondary endpoint.
     :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks.
         Defaults to 4*1024*1024, or 4MB.
-    :keyword int max_single_put_size: If the blob size is less than max_single_put_size, then the blob will be
+    :keyword int max_single_put_size: If the blob size is less than or equal max_single_put_size, then the blob will be
         uploaded with only one http PUT request. If the blob size is larger than max_single_put_size,
         the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB.
     :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py

@@ -56,7 +56,7 @@ async def upload_block_blob(  # pylint: disable=too-many-locals
         tier = kwargs.pop('standard_blob_tier', None)
 
         # Do single put if the size is smaller than config.max_single_put_size
-        if adjusted_count is not None and (adjusted_count < blob_settings.max_single_put_size):
+        if adjusted_count is not None and (adjusted_count <= blob_settings.max_single_put_size):
             try:
                 data = data.read(length)
                 if not isinstance(data, six.binary_type):