First shot of blob tests update

sdk/storage/azure-storage-blob/azure/storage/blob/blob_service_client.py

@@ -14,6 +14,8 @@
 except ImportError:
     from urlparse import urlparse # type: ignore
 
+from azure.core.paging import ItemPaged
+
 from ._shared.shared_access_signature import SharedAccessSignature
 from ._shared.models import LocationMode, Services
 from ._shared.utils import (
@@ -380,12 +382,11 @@ def set_service_properties(
     def list_containers(
             self, name_starts_with=None,  # type: Optional[str]
             include_metadata=False,  # type: Optional[bool]
-            marker=None,  # type: Optional[str]
             results_per_page=None,  # type: Optional[int]
             timeout=None,  # type: Optional[int]
             **kwargs
         ):
-        # type: (...) -> ContainerPropertiesPaged
+        # type: (...) -> Iterable[ContainerProperties]
         """Returns a generator to list the containers under the specified account.
 
         The generator will lazily follow the continuation tokens returned by
@@ -397,17 +398,13 @@ def list_containers(
         :param bool include_metadata:
             Specifies that container metadata be returned in the response.
             The default value is `False`.
-        :param str marker:
-            An opaque continuation token. This value can be retrieved from the
-            next_marker field of a previous generator object. If specified,
-            this generator will begin returning results from this point.
         :param int results_per_page:
             The maximum number of container names to retrieve per API
             call. If the request does not specify the server will return up to 5,000 items.
         :param int timeout:
             The timeout parameter is expressed in seconds.
         :returns: An iterable (auto-paging) of ContainerProperties.
-        :rtype: ~azure.core.blob.models.ContainerPropertiesPaged
+        :rtype: ~azure.core.paging.ItemPaged[~azure.core.blob.models.ContainerProperties]
 
         Example:
             .. literalinclude:: ../tests/test_blob_samples_service.py
@@ -424,8 +421,8 @@ def list_containers(
             include=include,
             timeout=timeout,
             **kwargs)
-        return ContainerPropertiesPaged(
-            command, prefix=name_starts_with, results_per_page=results_per_page, marker=marker)
+        return ItemPaged(
+            command, prefix=name_starts_with, results_per_page=results_per_page, page_iterator_class=ContainerPropertiesPaged)
 
     def create_container(
             self, name,  # type: str

sdk/storage/azure-storage-blob/azure/storage/blob/container_client.py

@@ -16,6 +16,8 @@
     from urlparse import urlparse # type: ignore
     from urllib2 import quote, unquote # type: ignore
 
+from azure.core.paging import ItemPaged
+
 import six
 
 from ._shared.shared_access_signature import BlobSharedAccessSignature
@@ -658,8 +660,8 @@ def set_container_access_policy(
         except StorageErrorException as error:
             process_storage_error(error)
 
-    def list_blobs(self, name_starts_with=None, include=None, marker=None, timeout=None, **kwargs):
-        # type: (Optional[str], Optional[Any], Optional[str], Optional[int], **Any) -> Iterable[BlobProperties]
+    def list_blobs(self, name_starts_with=None, include=None, timeout=None, **kwargs):
+        # type: (Optional[str], Optional[Any], Optional[int], **Any) -> Iterable[BlobProperties]
         """Returns a generator to list the blobs under the specified container.
         The generator will lazily follow the continuation tokens returned by
         the service.
@@ -670,14 +672,10 @@ def list_blobs(self, name_starts_with=None, include=None, marker=None, timeout=N
         :param list[str] include:
             Specifies one or more additional datasets to include in the response.
             Options include: 'snapshots', 'metadata', 'uncommittedblobs', 'copy', 'deleted'.
-        :param str marker:
-            An opaque continuation token. This value can be retrieved from the
-            next_marker field of a previous generator object. If specified,
-            this generator will begin returning results from this point.
         :param int timeout:
             The timeout parameter is expressed in seconds.
         :returns: An iterable (auto-paging) response of BlobProperties.
-        :rtype: ~azure.storage.blob.models.BlobPropertiesPaged
+        :rtype: ~azure.core.paging.ItemPaged[~azure.storage.blob.models.BlobProperties]
 
         Example:
             .. literalinclude:: ../tests/test_blob_samples_containers.py
@@ -696,7 +694,10 @@ def list_blobs(self, name_starts_with=None, include=None, marker=None, timeout=N
             include=include,
             timeout=timeout,
             **kwargs)
-        return BlobPropertiesPaged(command, prefix=name_starts_with, results_per_page=results_per_page, marker=marker)
+        return ItemPaged(
+            command, prefix=name_starts_with, results_per_page=results_per_page,
+            page_iterator_class=BlobPropertiesPaged)
+
 
     def walk_blobs(
             self, name_starts_with=None, # type: Optional[str]
@@ -723,32 +724,27 @@ def walk_blobs(
             element in the response body that acts as a placeholder for all blobs whose
             names begin with the same substring up to the appearance of the delimiter
             character. The delimiter may be a single character or a string.
-        :param str marker:
-            An opaque continuation token. This value can be retrieved from the
-            next_marker field of a previous generator object. If specified,
-            this generator will begin returning results from this point.
         :param int timeout:
             The timeout parameter is expressed in seconds.
         :returns: An iterable (auto-paging) response of BlobProperties.
-        :rtype: ~azure.storage.blob.models.BlobPrefix
+        :rtype: ~azure.core.paging.ItemPaged[~azure.storage.blob.models.BlobProperties]
         """
         if include and not isinstance(include, list):
             include = [include]
 
         results_per_page = kwargs.pop('results_per_page', None)
-        marker = kwargs.pop('marker', "")
         command = functools.partial(
             self._client.container.list_blob_hierarchy_segment,
             delimiter=delimiter,
             include=include,
             timeout=timeout,
             **kwargs)
-        return BlobPrefix(
+        return ItemPaged(
             command,
             prefix=name_starts_with,
             results_per_page=results_per_page,
-            marker=marker,
-            delimiter=delimiter)
+            delimiter=delimiter,
+            page_iterator_class=BlobPrefix)
 
     def upload_blob(
             self, name,  # type: Union[str, BlobProperties]

sdk/storage/azure-storage-blob/azure/storage/blob/models.py

@@ -9,7 +9,7 @@
 from enum import Enum
 from typing import List, Any, TYPE_CHECKING # pylint: disable=unused-import
 
-from azure.core.paging import Paged
+from azure.core.paging import PageIterator
 
 from ._shared.utils import (
     decode_base64,
@@ -287,14 +287,14 @@ def _from_generated(cls, generated):
         return props
 
 
-class ContainerPropertiesPaged(Paged):
+class ContainerPropertiesPaged(PageIterator):
     """An Iterable of Container properties.
 
     :ivar str service_endpoint: The service URL.
     :ivar str prefix: A container name prefix being used to filter the list.
     :ivar str current_marker: The continuation token of the current page of results.
     :ivar int results_per_page: The maximum number of results retrieved per API call.
-    :ivar str next_marker: The continuation token to retrieve the next page of results.
+    :ivar str continuation_token: The continuation token to retrieve the next page of results.
     :ivar str location_mode: The location mode being used to list results. The available
         options include "primary" and "secondary".
     :ivar current_page: The current page of listed results.
@@ -305,54 +305,48 @@ class ContainerPropertiesPaged(Paged):
         begin with the specified prefix.
     :param int results_per_page: The maximum number of container names to retrieve per
         call.
-    :param str marker: An opaque continuation token.
-    """
-    def __init__(self, command, prefix=None, results_per_page=None, marker=None):
-        super(ContainerPropertiesPaged, self).__init__(command, None)
+    :param str continuation_token: An opaque continuation token.
+    """
+    def __init__(self, command, prefix=None, results_per_page=None, continuation_token=None):
+        super(ContainerPropertiesPaged, self).__init__(
+            get_next=self._get_next_cb,
+            extract_data=self._extract_data_cb,
+            continuation_token=continuation_token or ""
+        )
+        self._command = command
         self.service_endpoint = None
         self.prefix = prefix
         self.current_marker = None
         self.results_per_page = results_per_page
-        self.next_marker = marker or ""
         self.location_mode = None
+        self.current_page = []
 
-    def _advance_page(self):
-        """Force moving the cursor to the next azure call.
-
-        This method is for advanced usage, iterator protocol is prefered.
-
-        :raises: StopIteration if no further page
-        :return: The current page list
-        :rtype: list
-        """
-        if self.next_marker is None:
-            raise StopIteration("End of paging")
-        self._current_page_iter_index = 0
+    def _get_next_cb(self, continuation_token):
         try:
-            self.location_mode, self._response = self._get_next(
-                marker=self.next_marker or None,
+            return self._command(
+                marker=continuation_token or None,
                 maxresults=self.results_per_page,
                 cls=return_context_and_deserialized,
                 use_location=self.location_mode)
         except StorageErrorException as error:
             process_storage_error(error)
 
+    def _extract_data_cb(self, get_next_return):
+        self.location_mode, self._response = get_next_return
         self.service_endpoint = self._response.service_endpoint
         self.prefix = self._response.prefix
         self.current_marker = self._response.marker
         self.results_per_page = self._response.max_results
-        self.current_page = self._response.container_items
-        self.next_marker = self._response.next_marker or None
-        return self.current_page
+        self.current_page = [self._build_item(item) for item in self._response.container_items]
+
+        return self._response.next_marker or None, self.current_page
 
-    def __next__(self): # type: ignore
-        item = super(ContainerPropertiesPaged, self).__next__()
+    @staticmethod
+    def _build_item(item):
         if isinstance(item, ContainerProperties):
             return item
         return ContainerProperties._from_generated(item)  # pylint: disable=protected-access
 
-    next = __next__
-
 
 class BlobProperties(DictMixin):
     """
@@ -469,14 +463,14 @@ def _from_generated(cls, generated):
         return blob
 
 
-class BlobPropertiesPaged(Paged):
+class BlobPropertiesPaged(PageIterator):
     """An Iterable of Blob properties.
 
     :ivar str service_endpoint: The service URL.
     :ivar str prefix: A blob name prefix being used to filter the list.
     :ivar str current_marker: The continuation token of the current page of results.
     :ivar int results_per_page: The maximum number of results retrieved per API call.
-    :ivar str next_marker: The continuation token to retrieve the next page of results.
+    :ivar str continuation_token: The continuation token to retrieve the next page of results.
     :ivar str location_mode: The location mode being used to list results. The available
         options include "primary" and "secondary".
     :ivar current_page: The current page of listed results.
@@ -489,7 +483,7 @@ class BlobPropertiesPaged(Paged):
         begin with the specified prefix.
     :param int results_per_page: The maximum number of blobs to retrieve per
         call.
-    :param str marker: An opaque continuation token.
+    :param str continuation_token: An opaque continuation token.
     :param str delimiter:
         Used to capture blobs whose names begin with the same substring up to
         the appearance of the delimiter character. The delimiter may be a single
@@ -503,55 +497,48 @@ def __init__(
             container=None,
             prefix=None,
             results_per_page=None,
-            marker=None,
+            continuation_token=None,
             delimiter=None,
             location_mode=None):
-        super(BlobPropertiesPaged, self).__init__(command, None)
+        super(BlobPropertiesPaged, self).__init__(
+            get_next=self._get_next_cb,
+            extract_data=self._extract_data_cb,
+            continuation_token=continuation_token or ""
+        )
+        self._command = command
         self.service_endpoint = None
         self.prefix = prefix
         self.current_marker = None
         self.results_per_page = results_per_page
-        self.next_marker = marker or ""
         self.container = container
         self.delimiter = delimiter
         self.current_page = None
         self.location_mode = location_mode
 
-    def _advance_page(self):
-        """Force moving the cursor to the next azure call.
-
-        This method is for advanced usage, iterator protocol is prefered.
-
-        :raises: StopIteration if no further page
-        :return: The current page list
-        :rtype: list
-        """
-        if self.next_marker is None:
-            raise StopIteration("End of paging")
-        self._current_page_iter_index = 0
-
+    def _get_next_cb(self, continuation_token):
         try:
-            self.location_mode, self._response = self._get_next(
+            return self._command(
                 prefix=self.prefix,
-                marker=self.next_marker or None,
+                marker=continuation_token or None,
                 maxresults=self.results_per_page,
                 cls=return_context_and_deserialized,
                 use_location=self.location_mode)
         except StorageErrorException as error:
             process_storage_error(error)
 
+    def _extract_data_cb(self, get_next_return):
+        self.location_mode, self._response = get_next_return
         self.service_endpoint = self._response.service_endpoint
         self.prefix = self._response.prefix
         self.current_marker = self._response.marker
         self.results_per_page = self._response.max_results
-        self.current_page = self._response.segment.blob_items
-        self.next_marker = self._response.next_marker or None
         self.container = self._response.container_name
+        self.current_page = [self._build_item(item) for item in self._response.segment.blob_items]
         self.delimiter = self._response.delimiter
-        return self.current_page
 
-    def __next__(self):
-        item = super(BlobPropertiesPaged, self).__next__()
+        return self._response.next_marker or None, self.current_page
+
+    def _build_item(self, item):
         if isinstance(item, BlobProperties):
             return item
         if isinstance(item, BlobItem):
@@ -560,8 +547,6 @@ def __next__(self):
             return blob
         return item
 
-    next = __next__
-
 
 class BlobPrefix(BlobPropertiesPaged, DictMixin):
     """An Iterable of Blob properties.
@@ -600,36 +585,15 @@ def __init__(self, *args, **kwargs):
         super(BlobPrefix, self).__init__(*args, **kwargs)
         self.name = self.prefix
 
-    def _advance_page(self):
-        """Force moving the cursor to the next azure call.
-
-        This method is for advanced usage, iterator protocol is prefered.
-
-        :raises: StopIteration if no further page
-        :return: The current page list
-        :rtype: list
-        """
-        if self.next_marker is None:
-            raise StopIteration("End of paging")
-        self._current_page_iter_index = 0
-        self.location_mode, self._response = self._get_next(
-            prefix=self.prefix,
-            marker=self.next_marker or None,
-            maxresults=self.results_per_page,
-            cls=return_context_and_deserialized,
-            use_location=self.location_mode)
-        self.service_endpoint = self._response.service_endpoint
-        self.prefix = self._response.prefix
-        self.current_marker = self._response.marker
-        self.results_per_page = self._response.max_results
-        self.current_page = self._response.segment.blob_prefixes
-        self.current_page.extend(self._response.segment.blob_items)
-        self.next_marker = self._response.next_marker or None
-        self.container = self._response.container_name
-        self.delimiter = self._response.delimiter
+    def _extract_data_cb(self, get_next_return):
+        continuation_token, _ = super(BlobPrefix, self)._extract_data_cb(get_next_return)
+        self.current_page = self._response.segment.blob_items + self._response.segment.blob_prefixes
+        self.current_page = [self._build_item(item) for item in self.current_page]
 
-    def __next__(self):
-        item = super(BlobPrefix, self).__next__()
+        return continuation_token, self.current_page
+
+    def _build_item(self, item):
+        item = super(BlobPrefix, self)._build_item(item)
         if isinstance(item, GenBlobPrefix):
             return BlobPrefix(
                 self._get_next,
@@ -639,8 +603,6 @@ def __next__(self):
                 location_mode=self.location_mode)
         return item
 
-    next = __next__
-
 
 class LeaseProperties(DictMixin):
     """Blob Lease Properties.