Merging from master

.gitignore

@@ -109,6 +109,5 @@ doc/build/html/index.html
 # Windows specific leftover:
 doc/tmp.sv
 doc/source/styled.xlsx
-doc/source/templates/
 env/
 doc/source/savefig/

.pep8speaks.yml

@@ -13,7 +13,6 @@ pycodestyle:
         - W503,  # line break before binary operator
         - W504,  # line break after binary operator
         - E402,  # module level import not at top of file
-        - E722,  # do not use bare except
         - E731,  # do not assign a lambda expression, use a def
         - C406,  # Unnecessary list literal - rewrite as a dict literal.
         - C408,  # Unnecessary dict call - rewrite as a literal.

.travis.yml

@@ -23,7 +23,7 @@ env:
 
 git:
     # for cloning
-    depth: 1000
+    depth: 1500
 
 matrix:
     fast_finish: true

asv_bench/benchmarks/timeseries.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+import dateutil
 import numpy as np
 from pandas import to_datetime, date_range, Series, DataFrame, period_range
 from pandas.tseries.frequencies import infer_freq
@@ -57,7 +58,10 @@ def time_to_pydatetime(self, index_type):
 
 class TzLocalize(object):
 
-    def setup(self):
+    params = [None, 'US/Eastern', 'UTC', dateutil.tz.tzutc()]
+    param_names = 'tz'
+
+    def setup(self, tz):
         dst_rng = date_range(start='10/29/2000 1:00:00',
                              end='10/29/2000 1:59:59', freq='S')
         self.index = date_range(start='10/29/2000',
@@ -68,8 +72,8 @@ def setup(self):
                                                   end='10/29/2000 3:00:00',
                                                   freq='S'))
 
-    def time_infer_dst(self):
-        self.index.tz_localize('US/Eastern', ambiguous='infer')
+    def time_infer_dst(self, tz):
+        self.index.tz_localize(tz, ambiguous='infer')
 
 
 class ResetIndex(object):
@@ -377,15 +381,35 @@ def time_dup_string_tzoffset_dates(self, cache):
 
 class DatetimeAccessor(object):
 
-    def setup(self):
+    params = [None, 'US/Eastern', 'UTC', dateutil.tz.tzutc()]
+    param_names = 'tz'
+
+    def setup(self, tz):
         N = 100000
-        self.series = Series(date_range(start='1/1/2000', periods=N, freq='T'))
+        self.series = Series(
+            date_range(start='1/1/2000', periods=N, freq='T', tz=tz)
+        )
 
-    def time_dt_accessor(self):
+    def time_dt_accessor(self, tz):
         self.series.dt
 
-    def time_dt_accessor_normalize(self):
+    def time_dt_accessor_normalize(self, tz):
         self.series.dt.normalize()
 
+    def time_dt_accessor_month_name(self, tz):
+        self.series.dt.month_name()
+
+    def time_dt_accessor_day_name(self, tz):
+        self.series.dt.day_name()
+
+    def time_dt_accessor_time(self, tz):
+        self.series.dt.time
+
+    def time_dt_accessor_date(self, tz):
+        self.series.dt.date
+
+    def time_dt_accessor_year(self, tz):
+        self.series.dt.year
+
 
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/timestamp.py

@@ -2,6 +2,7 @@
 
 from pandas import Timestamp
 import pytz
+import dateutil
 
 
 class TimestampConstruction(object):
@@ -29,7 +30,8 @@ def time_fromtimestamp(self):
 
 
 class TimestampProperties(object):
-    _tzs = [None, pytz.timezone('Europe/Amsterdam')]
+    _tzs = [None, pytz.timezone('Europe/Amsterdam'), pytz.UTC,
+            dateutil.tz.tzutc()]
     _freqs = [None, 'B']
     params = [_tzs, _freqs]
     param_names = ['tz', 'freq']
@@ -87,7 +89,8 @@ def time_microsecond(self, tz, freq):
 
 
 class TimestampOps(object):
-    params = [None, 'US/Eastern']
+    params = [None, 'US/Eastern', pytz.UTC,
+              dateutil.tz.tzutc()]
     param_names = ['tz']
 
     def setup(self, tz):
@@ -102,6 +105,17 @@ def time_replace_None(self, tz):
     def time_to_pydatetime(self, tz):
         self.ts.to_pydatetime()
 
+    def time_normalize(self, tz):
+        self.ts.normalize()
+
+    def time_tz_convert(self, tz):
+        if self.ts.tz is not None:
+            self.ts.tz_convert(tz)
+
+    def time_tz_localize(self, tz):
+        if self.ts.tz is None:
+            self.ts.tz_localize(tz)
+
 
 class TimestampAcrossDst(object):
     def setup(self):

ci/deps/travis-36-doc.yaml

@@ -8,10 +8,11 @@ dependencies:
   - bottleneck
   - cython>=0.28.2
   - fastparquet
+  - gitpython
   - html5lib
   - hypothesis>=3.58.0
   - ipykernel
-  - ipython==6.5.0
+  - ipython
   - ipywidgets
   - lxml
   - matplotlib

ci/deps/travis-36.yaml

@@ -9,7 +9,7 @@ dependencies:
   - fastparquet
   - flake8>=3.5
   - flake8-comprehensions
-  - flake8-rst
+  - flake8-rst=0.4.2
   - gcsfs
   - geopandas
   - html5lib

doc/make.py

@@ -126,7 +126,12 @@ def _process_single_doc(self, single_doc):
             self.single_doc = 'api'
         elif os.path.exists(os.path.join(SOURCE_PATH, single_doc)):
             self.single_doc_type = 'rst'
-            self.single_doc = os.path.splitext(os.path.basename(single_doc))[0]
+
+            if 'whatsnew' in single_doc:
+                basename = single_doc
+            else:
+                basename = os.path.basename(single_doc)
+            self.single_doc = os.path.splitext(basename)[0]
         elif os.path.exists(
                 os.path.join(SOURCE_PATH, '{}.rst'.format(single_doc))):
             self.single_doc_type = 'rst'

doc/source/api.rst

@@ -1724,6 +1724,7 @@ MultiIndex Components
    MultiIndex.set_levels
    MultiIndex.set_labels
    MultiIndex.to_hierarchical
+   MultiIndex.to_flat_index
    MultiIndex.to_frame
    MultiIndex.is_lexsorted
    MultiIndex.sortlevel

doc/source/conf.py

@@ -40,7 +40,6 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 # sys.path.append(os.path.abspath('.'))
 sys.path.insert(0, os.path.abspath('../sphinxext'))
-
 sys.path.extend([
 
     # numpy standard doc extensions
@@ -75,6 +74,7 @@
               'sphinx.ext.ifconfig',
               'sphinx.ext.linkcode',
               'nbsphinx',
+              'contributors',  # custom pandas extension
               ]
 
 try:
@@ -120,7 +120,9 @@
 templates_path = ['../_templates']
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = [
+    '.rst',
+]
 
 # The encoding of source files.
 source_encoding = 'utf-8'
@@ -298,8 +300,26 @@
     for page in moved_api_pages
 }
 
+
+common_imports = """\
+.. currentmodule:: pandas
+
+.. ipython:: python
+   :suppress:
+
+   import numpy as np
+   from pandas import *
+   import pandas as pd
+   randn = np.random.randn
+   np.set_printoptions(precision=4, suppress=True)
+   options.display.max_rows = 15
+   from pandas.compat import StringIO
+"""
+
+
 html_context = {
-    'redirects': {old: new for old, new in moved_api_pages}
+    'redirects': {old: new for old, new in moved_api_pages},
+    'common_imports': common_imports,
 }
 
 # If false, no module index is generated.
@@ -388,6 +408,7 @@
                         category=FutureWarning)
 
 
+ipython_warning_is_error = False
 ipython_exec_lines = [
     'import numpy as np',
     'import pandas as pd',
@@ -565,7 +586,7 @@ def linkcode_resolve(domain, info):
     for part in fullname.split('.'):
         try:
             obj = getattr(obj, part)
-        except:
+        except AttributeError:
             return None
 
     try:
@@ -574,14 +595,14 @@ def linkcode_resolve(domain, info):
             fn = inspect.getsourcefile(inspect.unwrap(obj))
         else:
             fn = inspect.getsourcefile(obj)
-    except:
+    except TypeError:
         fn = None
     if not fn:
         return None
 
     try:
         source, lineno = inspect.getsourcelines(obj)
-    except:
+    except OSError:
         lineno = None
 
     if lineno:
@@ -653,7 +674,23 @@ def process_class_docstrings(app, what, name, obj, options, lines):
 ]
 
 
+def rstjinja(app, docname, source):
+    """
+    Render our pages as a jinja template for fancy templating goodness.
+    """
+    # http://ericholscher.com/blog/2016/jul/25/integrating-jinja-rst-sphinx/
+    # Make sure we're outputting HTML
+    if app.builder.format != 'html':
+        return
+    src = source[0]
+    rendered = app.builder.templates.render_string(
+        src, app.config.html_context
+    )
+    source[0] = rendered
+
+
 def setup(app):
+    app.connect("source-read", rstjinja)
     app.connect("autodoc-process-docstring", remove_flags_docstring)
     app.connect("autodoc-process-docstring", process_class_docstrings)
     app.add_autodocumenter(AccessorDocumenter)

doc/source/contributing.rst

@@ -591,21 +591,14 @@ run this slightly modified command::
 
    git diff master --name-only -- "*.py" | grep "pandas/" | xargs flake8
 
-Note that on Windows, these commands are unfortunately not possible because
-commands like ``grep`` and ``xargs`` are not available natively. To imitate the
-behavior with the commands above, you should run::
+Windows does not support the ``grep`` and ``xargs`` commands (unless installed
+for example via the `MinGW <http://www.mingw.org/>`__ toolchain), but one can
+imitate the behaviour as follows::
 
-    git diff master --name-only -- "*.py"
+    for /f %i in ('git diff upstream/master --name-only ^| findstr pandas/') do flake8 %i
 
-This will list all of the Python files that have been modified. The only ones
-that matter during linting are any whose directory filepath begins with "pandas."
-For each filepath, copy and paste it after the ``flake8`` command as shown below:
-
-    flake8 <python-filepath>
-
-Alternatively, you can install the ``grep`` and ``xargs`` commands via the
-`MinGW <http://www.mingw.org/>`__ toolchain, and it will allow you to run the
-commands above.
+This will also get all the files being changed by the PR (and within the
+``pandas/`` folder), and run ``flake8`` on them one after the other.
 
 .. _contributing.import-formatting:
 
@@ -1103,7 +1096,7 @@ Information on how to write a benchmark and how to use asv can be found in the
 Documenting your code
 ---------------------
 
-Changes should be reflected in the release notes located in ``doc/source/whatsnew/vx.y.z.txt``.
+Changes should be reflected in the release notes located in ``doc/source/whatsnew/vx.y.z.rst``.
 This file contains an ongoing change log for each release.  Add an entry to this file to
 document your fix, enhancement or (unavoidable) breaking change.  Make sure to include the
 GitHub issue number when adding your entry (using ``:issue:`1234``` where ``1234`` is the

doc/source/index.rst.template

@@ -118,7 +118,7 @@ See the package overview for more detail about what's in the library.
     {{ single_doc }}
     {% endif -%}
     {% if not single_doc -%}
-    whatsnew
+    What's New <whatsnew/v0.24.0>
     install
     contributing
     overview
@@ -159,5 +159,5 @@ See the package overview for more detail about what's in the library.
     developer
     internals
     extending
-    release
+    releases
     {% endif -%}

doc/source/install.rst

@@ -286,7 +286,9 @@ Optional Dependencies
   `xsel <http://www.vergenet.net/~conrad/software/xsel/>`__, or
   `xclip <https://github.com/astrand/xclip/>`__: necessary to use
   :func:`~pandas.read_clipboard`. Most package managers on Linux distributions will have ``xclip`` and/or ``xsel`` immediately available for installation.
-* `pandas-gbq <https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies>`__: for Google BigQuery I/O.
+* `pandas-gbq
+  <https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies>`__:
+  for Google BigQuery I/O. (pandas-gbq >= 0.8.0)
 
 
 * `Backports.lzma <https://pypi.org/project/backports.lzma/>`__: Only for Python 2, for writing to and/or reading from an xz compressed DataFrame in CSV; Python 3 support is built into the standard library.

doc/source/io.rst

@@ -1580,12 +1580,19 @@ You can pass in a URL to a CSV file:
    df = pd.read_csv('https://download.bls.gov/pub/time.series/cu/cu.item',
                     sep='\t')
 
-S3 URLs are handled as well:
+S3 URLs are handled as well but require installing the `S3Fs
+<https://pypi.org/project/s3fs/>`_ library:
 
 .. code-block:: python
 
    df = pd.read_csv('s3://pandas-test/tips.csv')
 
+If your S3 bucket requires cedentials you will need to set them as environment
+variables or in the ``~/.aws/credentials`` config file, refer to the `S3Fs
+documentation on credentials
+<https://s3fs.readthedocs.io/en/latest/#credentials>`_.
+
+
 
 Writing out Data
 ''''''''''''''''