Extend MemoryFile and ZipMemoryFile

docs/manual.rst

@@ -1287,11 +1287,11 @@ distributed with Fiona.
 
 .. sourcecode:: console
 
-  $ zip /tmp/zed.zip docs/data/test_uk.*
-  adding: docs/data/test_uk.shp (deflated 48%)
-  adding: docs/data/test_uk.shx (deflated 37%)
-  adding: docs/data/test_uk.dbf (deflated 98%)
-  adding: docs/data/test_uk.prj (deflated 15%)
+  $ zip /tmp/coutwildrnp.zip tests/data/coutwildrnp.dbf tests/data/coutwildrnp.prj tests/data/coutwildrnp.shp tests/data/coutwildrnp.shx
+  adding: tests/data/coutwildrnp.dbf (deflated 94%)
+  adding: tests/data/coutwildrnp.prj (deflated 15%)
+  adding: tests/data/coutwildrnp.shp (deflated 62%)
+  adding: tests/data/coutwildrnp.shx (deflated 34%)
 
 The `vfs` keyword parameter for :py:func:`fiona.listlayers` and
 :py:func:`fiona.open` may be an Apache Commons VFS style string beginning with
@@ -1302,19 +1302,78 @@ absolute path within that archive. The layers in that Zip archive are:
 .. sourcecode:: pycon
 
   >>> import fiona
-  >>> fiona.listlayers('/docs/data', vfs='zip:///tmp/zed.zip')
-  ['test_uk']
+  >>> fiona.listlayers('zip:///tmp/coutwildrnp.zip/tests/data/coutwildrnp.shp')
+  ['coutwildrnp']
 
 The single shapefile may also be accessed like so:
 
 .. sourcecode:: pycon
 
-  >>> with fiona.open(
-  ...         '/docs/data/test_uk.shp', 
-  ...         vfs='zip:///tmp/zed.zip') as c:
-  ...     print(len(c))
-  ...
-  48
+    >>> with fiona.open('zip:///tmp/coutwildrnp.zip/tests/data/coutwildrnp.shp') as c:
+    ...     print(len(c))
+    ...
+    67
+
+The files of an archive can be listed with :py:func:`fiona.listdir`.
+
+.. sourcecode:: pycon
+
+    >>> fiona.listdir('zip:///tmp/coutwildrnp.zip/tests/data')
+    ['coutwildrnp.dbf', 'coutwildrnp.prj', 'coutwildrnp.shp', 'coutwildrnp.shx']
+
+
+MemoryFile and ZipMemoryFile
+----------------------------
+
+:py:class:`fiona.io.MemoryFile` and :py:class:`fiona.io.ZipMemoryFile` allow
+formatted feature collections, even zipped feature collections, to be read or
+written in memory, with no filesystem access required. For example, you may
+have a zipped shapefile in a stream of bytes coming from a web upload or
+download.
+
+.. code-block:: pycon
+
+    >>> data = open('tests/data/coutwildrnp.zip', 'rb').read()
+    >>> len(data)
+    154006
+    >>> data[:20]
+    b'PK\x03\x04\x14\x00\x00\x00\x00\x00\xaa~VM\xech\xae\x1e\xec\xab'
+
+The feature collection in this stream of bytes can be accessed by wrapping it
+in an instance of ZipMemoryFile.
+
+.. code-block:: pycon
+
+    >>> from fiona.io import ZipMemoryFile
+    >>> with ZipMemoryFile(data) as zip:
+    ...     with zip.open('coutwildrnp.shp') as collection:
+    ...         print(len(collection))
+    ...         print(collection.schema)
+    ...
+    67
+    {'properties': OrderedDict([('PERIMETER', 'float:24.15'), ('FEATURE2', 'str:80'), ('NAME', 'str:80'), ('FEATURE1', 'str:80'), ('URL', 'str:101'), ('AGBUR', 'str:80'), ('AREA', 'float:24.15'), ('STATE_FIPS', 'str:80'), ('WILDRNP020', 'int:10'), ('STATE', 'str:80')]), 'geometry': 'Polygon'}
+
+*New in 1.8.0*
+
+Files in a ZipMemoryFile can be listed using `listdir()`.
+
+.. code-block:: pycon
+
+    >>> from fiona.io import ZipMemoryFile
+    >>> with ZipMemoryFile(data) as zip:
+    ...     print(zip.listdir('/'))
+    ...
+    ['coutwildrnp.shp', 'coutwildrnp.shx', 'coutwildrnp.dbf', 'coutwildrnp.prj']
+
+
+Similarly, layers can be listed using `listlayers()`.
+
+.. code-block:: pycon
+
+    >>> with ZipMemoryFile(data) as zip:
+    ...     print(zip.listlayers('/coutwildrnp.shp'))
+    ...
+    ['coutwildrnp']
 
 
 Unsupported drivers

fiona/__init__.py

@@ -81,6 +81,7 @@ class Path:
     libdir = os.path.join(os.path.dirname(__file__), ".libs")
     os.environ["PATH"] = os.environ["PATH"] + ";" + libdir
 
+
 import fiona._loading
 with fiona._loading.add_gdal_dll_directories():
     from fiona.collection import BytesCollection, Collection
@@ -93,7 +94,7 @@ class Path:
         get_gdal_version_tuple)
     from fiona.compat import OrderedDict
     from fiona.io import MemoryFile
-    from fiona.ogrext import _bounds, _listlayers, FIELD_TYPES_MAP, _remove, _remove_layer
+    from fiona.ogrext import _bounds, _listlayers, _listdir, FIELD_TYPES_MAP, _remove, _remove_layer
     from fiona.path import ParsedPath, parse_path, vsi_path
     from fiona.vfs import parse_paths as vfs_parse_paths
     from fiona._show_versions import show_versions
@@ -104,7 +105,7 @@ class Path:
     import uuid
 
 
-__all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
+__all__ = ['bounds', 'listlayers', 'listdir', 'open', 'prop_type', 'prop_width']
 __version__ = "1.8.13.post1"
 __gdal_version__ = get_gdal_release_name()
 
@@ -331,7 +332,7 @@ def listlayers(fp, vfs=None):
     if hasattr(fp, 'read'):
 
         with MemoryFile(fp.read()) as memfile:
-            return  _listlayers(memfile.name)
+            return _listlayers(memfile.name)
 
     else:
 
@@ -354,6 +355,33 @@ def listlayers(fp, vfs=None):
         return _listlayers(vsi_path(pobj))
 
 
+@ensure_env_with_credentials
+def listdir(path):
+
+    """List files in a directory
+
+    Parameters
+    ----------
+    path : URI (str or pathlib.Path)
+        A dataset resource identifier.
+
+    Returns
+    -------
+    list
+        A list of filename strings.
+
+    """
+
+    if isinstance(path, Path):
+        path = str(path)
+
+    if not isinstance(path, string_types):
+        raise TypeError("invalid path: %r" % path)
+
+    pobj = parse_path(path)
+    return _listdir(vsi_path(pobj))
+
+
 def prop_width(val):
     """Returns the width of a str type property.
 

fiona/drvsupport.py

@@ -2,7 +2,6 @@
 from fiona.env import Env
 from fiona._env import get_gdal_version_num, calc_gdal_version_num
 
-
 # Here is the list of available drivers as (name, modes) tuples. Currently,
 # we only expose the defaults (excepting FileGDB). We also don't expose
 # the CSV or GeoJSON drivers. Use Python's csv and json modules instead.
@@ -366,3 +365,78 @@ def _driver_supports_unknown_timezones(driver, field_type):
         elif get_gdal_version_num() < calc_gdal_version_num(*_drivers_not_supporting_unknown_timezone[field_type][driver]):
             return False
     return True
+
+
+# _zip_memoryfile_not_supported['/vsizip/']['w']['GMT'] = (2, 0, 0): write mode is not supported for '/vsizip/'
+# before GDAL 2.0
+# _zip_memoryfile_not_supported['/vsizip/']['w']['GPKG'] = None: write mode is not supported for '/vsizip/' for all
+# versions of GDAL
+#
+# Reasons for missing support:
+# BNA: segfault with GDAL 1.x
+# DGN: Failed to open output file (<= GDAL 2.2), segfault (GDAL > 2.2)
+# GPKG,DXF,ESRI Shapefile,GPX,MapInfo File,PCIDSK': Random access not supported for writable file in /vsizip
+# GMT: Random access not supported for /vsizip for gdal 1.x
+# GPSTrackMaker: VSIFSeekL() is not supported on writable Zip files
+_zip_memoryfile_not_supported = {
+    '/vsizip/': {
+        'w': {
+            'BNA': (2, 0, 0),
+            'GMT': None,
+            'OGR_GMT': None,
+            'DGN': None,
+            'GPKG': None,
+            'DXF': None,
+            'ESRI Shapefile': None,
+            'GPX': None,
+            'MapInfo File': None,
+            'PCIDSK': None,
+            'GPSTrackMaker': None,
+            'FileGDB': None
+        }
+    }
+}
+
+
+def _zip_memoryfile_supports_mode(vsi, driver, mode):
+    """ Returns True if mode is supported for /vsizip/vsimem/"""
+
+    if (vsi in _zip_memoryfile_not_supported and mode in _zip_memoryfile_not_supported[vsi] and driver in
+            _zip_memoryfile_not_supported[vsi][mode]):
+        if _zip_memoryfile_not_supported[vsi][mode][driver] is None:
+            return False
+        elif get_gdal_version_num() < calc_gdal_version_num(*_zip_memoryfile_not_supported[vsi][mode][driver]):
+            return False
+
+    return True
+
+# 'BNA': (2, 0, 0): BNA driver supports MemoryFile access mode starting with GDAL 2.0.0
+# 'FileGDB': None: FileGDB driver never supports MemoryFile access mode
+_memoryfile_not_supported = {
+    'w': {
+        'BNA': (2, 0, 0),
+        'DGN': (2, 3, 0),
+        'GPKG': (2, 0, 0),
+        'MapInfo File': (2, 0, 0),
+        'FileGDB': None
+    },
+    'a': {
+        'BNA': (2, 0, 0),
+        'DGN': (2, 3, 0),
+        # Test fails for GPKG with sqlite3_open(/vsimem/...) failed: out of memory for gdal 1.x
+        'GPKG': (2, 0, 0),
+        'MapInfo File': (2, 0, 0),
+        'PCIDSK': None,
+        'FileGDB': None
+    }
+}
+
+
+def _memoryfile_supports_mode(driver, mode):
+    """ Returns True if mode is supported for /vsimem/"""
+    if mode in _memoryfile_not_supported and driver in _memoryfile_not_supported[mode]:
+        if _memoryfile_not_supported[mode][driver] is None:
+            return False
+        elif get_gdal_version_num() < calc_gdal_version_num(*_memoryfile_not_supported[mode][driver]):
+            return False
+    return True